honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add repository-url module and move URLRepository (#22752) 

This is related to #22116. URLRepository requires SocketPermission
connect. This commit introduces a new module called "repository-url"
where URLRepository will reside. With the new module, permissions can
be removed from core.
This commit is contained in:
Tim Brooks 2017-01-25 17:09:25 -06:00 committed by GitHub
parent e9a68b3287
commit 719e75bb3f
21 changed files with 525 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
View File

@ -276,8 +276,7 @@ class ClusterFormationTasks {
'path.repo' : "${node.sharedDir}/repo",
'path.shared_data' : "${node.sharedDir}/",
// Define a node attribute so we can test that it exists
'node.attr.testattr' : 'test',
'repositories.url.allowed_urls': 'http://snapshot.test*'
'node.attr.testattr' : 'test'
]
// we set min master nodes to the total number of nodes in the cluster and
// basically skip initial state recovery to allow the cluster to form using a realistic master election

2
buildSrc/src/main/resources/checkstyle_suppressions.xml
View File

@ -256,7 +256,6 @@
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]Base64.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]Numbers.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]fs[/\\]FsBlobStore.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]url[/\\]URLBlobStore.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]bytes[/\\]BytesArray.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]bytes[/\\]PagedBytesReference.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cache[/\\]Cache.java" checks="LineLength" />
@ -437,7 +436,6 @@
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]blobstore[/\\]ChecksumBlobStoreFormat.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]fs[/\\]FsRepository.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]uri[/\\]URLIndexShardRepository.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]uri[/\\]URLRepository.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]RestController.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestCountAction.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestIndicesAction.java" checks="LineLength" />

4
core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
View File

@ -81,7 +81,6 @@ import org.elasticsearch.monitor.process.ProcessService;
import org.elasticsearch.node.Node;
import org.elasticsearch.plugins.PluginsService;
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.repositories.uri.URLRepository;
import org.elasticsearch.rest.BaseRestHandler;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.SearchModule;
@ -348,9 +347,6 @@ public final class ClusterSettings extends AbstractScopedSettings {
Node.NODE_INGEST_SETTING,
Node.NODE_ATTRIBUTES,
Node.NODE_LOCAL_STORAGE_SETTING,
URLRepository.ALLOWED_URLS_SETTING,
URLRepository.REPOSITORIES_URL_SETTING,
URLRepository.SUPPORTED_PROTOCOLS_SETTING,
TransportMasterNodeReadAction.FORCE_LOCAL_SETTING,
AutoCreateIndex.AUTO_CREATE_INDEX_SETTING,
BaseRestHandler.MULTI_ALLOW_EXPLICIT_INDEX,

13
core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
View File

@ -19,24 +19,22 @@
package org.elasticsearch.repositories;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.elasticsearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.binder.LinkedBindingBuilder;
import org.elasticsearch.common.inject.multibindings.MapBinder;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.env.Environment;
import org.elasticsearch.plugins.RepositoryPlugin;
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.repositories.uri.URLRepository;
import org.elasticsearch.snapshots.RestoreService;
import org.elasticsearch.snapshots.SnapshotShardsService;
import org.elasticsearch.snapshots.SnapshotsService;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Sets up classes for Snapshot/Restore.
*/
@ -47,7 +45,6 @@ public class RepositoriesModule extends AbstractModule {
public RepositoriesModule(Environment env, List<RepositoryPlugin> repoPlugins, NamedXContentRegistry namedXContentRegistry) {
Map<String, Repository.Factory> factories = new HashMap<>();
factories.put(FsRepository.TYPE, (metadata) -> new FsRepository(metadata, env, namedXContentRegistry));
factories.put(URLRepository.TYPE, (metadata) -> new URLRepository(metadata, env, namedXContentRegistry));
for (RepositoryPlugin repoPlugin : repoPlugins) {
Map<String, Repository.Factory> newRepoTypes = repoPlugin.getRepositories(env, namedXContentRegistry);

48
core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
View File

@ -26,21 +26,22 @@ import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
import org.elasticsearch.common.io.FileTestUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.repositories.uri.URLRepository;
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
import org.elasticsearch.snapshots.RestoreInfo;
import org.elasticsearch.snapshots.SnapshotInfo;
import org.elasticsearch.snapshots.SnapshotRestoreException;
import org.elasticsearch.snapshots.mockstore.MockRepository;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.elasticsearch.test.VersionUtils;
import org.junit.BeforeClass;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
@ -61,27 +62,19 @@ import static org.hamcrest.Matchers.notNullValue;
@ClusterScope(scope = Scope.TEST)
public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase {
private static Path repoPath;
@Override
protected Settings nodeSettings(int nodeOrdinal) {
if (randomBoolean()) {
// Configure using path.repo
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(Environment.PATH_REPO_SETTING.getKey(), getBwcIndicesPath())
.build();
} else {
// Configure using url white list
try {
URI repoJarPatternUri = new URI("jar:" + getBwcIndicesPath().toUri().toString() + "*.zip!/repo/");
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.putArray(URLRepository.ALLOWED_URLS_SETTING.getKey(), repoJarPatternUri.toString())
.build();
} catch (URISyntaxException ex) {
throw new IllegalArgumentException(ex);
}
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(Environment.PATH_REPO_SETTING.getKey(), repoPath)
.build();
}
}
@BeforeClass
public static void repoSetup() throws IOException {
repoPath = createTempDir("repositories");
}
public void testRestoreOldSnapshots() throws Exception {
@ -151,13 +144,14 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase {
}
private void createRepo(String prefix, String version, String repo) throws Exception {
Path repoFile = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip");
URI repoFileUri = repoFile.toUri();
URI repoJarUri = new URI("jar:" + repoFileUri.toString() + "!/repo/");
Path repoFileFromBuild = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip");
String repoFileName = repoFileFromBuild.getFileName().toString().split(".zip")[0];
Path fsRepoPath = repoPath.resolve(repoFileName);
FileTestUtils.unzip(repoFileFromBuild, fsRepoPath, null);
logger.info("--> creating repository [{}] for version [{}]", repo, version);
assertAcked(client().admin().cluster().preparePutRepository(repo)
.setType("url").setSettings(Settings.builder()
.put("url", repoJarUri.toString())));
.setType(MockRepository.TYPE).setSettings(Settings.builder()
.put(FsRepository.REPOSITORIES_LOCATION_SETTING.getKey(), fsRepoPath.getParent().relativize(fsRepoPath).resolve("repo").toString())));
}
private void testOldSnapshot(String version, String repo, String snapshot) throws IOException {
@ -198,7 +192,7 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase {
equalTo("{\"type1\":{\"_source\":{\"enabled\":0}}}"),
equalTo("{\"type1\":{\"_source\":{\"enabled\":\"off\"}}}"),
equalTo("{\"type1\":{\"_source\":{\"enabled\":\"no\"}}}")
));
));
assertThat(template.aliases().size(), equalTo(3));
assertThat(template.aliases().get("alias1"), notNullValue());
assertThat(template.aliases().get("alias2").filter().string(), containsString(version));

24
core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
View File

@ -142,30 +142,6 @@ public class RepositoriesIT extends AbstractSnapshotIntegTestCase {
} catch (RepositoryException ex) {
assertThat(ex.toString(), containsString("location [" + location + "] doesn't match any of the locations specified by path.repo"));
}
String repoUrl = invalidRepoPath.toAbsolutePath().toUri().toURL().toString();
String unsupportedUrl = repoUrl.replace("file:/", "netdoc:/");
logger.info("--> trying creating url repository with unsupported url protocol");
try {
client().admin().cluster().preparePutRepository("test-repo")
.setType("url").setSettings(Settings.builder().put("url", unsupportedUrl))
.get();
fail("Shouldn't be here");
} catch (RepositoryException ex) {
assertThat(ex.toString(),
either(containsString("unsupported url protocol [netdoc]"))
.or(containsString("unknown protocol: netdoc"))); // newer versions of JDK 9
}
logger.info("--> trying creating url repository with location that is not registered in path.repo setting");
try {
client().admin().cluster().preparePutRepository("test-repo")
.setType("url").setSettings(Settings.builder().put("url", invalidRepoPath.toUri().toURL()))
.get();
fail("Shouldn't be here");
} catch (RepositoryException ex) {
assertThat(ex.toString(), containsString("doesn't match any of the locations specified by path.repo"));
}
}
public void testRepositoryAckTimeout() throws Exception {

57
core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
View File

@ -1442,63 +1442,6 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
}
public void testUrlRepository() throws Exception {
Client client = client();
logger.info("--> creating repository");
Path repositoryLocation = randomRepoPath();
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType("fs").setSettings(Settings.builder()
.put("location", repositoryLocation)
.put("compress", randomBoolean())
.put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
createIndex("test-idx");
ensureGreen();
logger.info("--> indexing some data");
for (int i = 0; i < 100; i++) {
index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
}
refresh();
assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
logger.info("--> snapshot");
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get();
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
logger.info("--> delete index");
cluster().wipeIndices("test-idx");
logger.info("--> create read-only URL repository");
assertAcked(client.admin().cluster().preparePutRepository("url-repo")
.setType("url").setSettings(Settings.builder()
.put("url", repositoryLocation.toUri().toURL())
.put("list_directories", randomBoolean())));
logger.info("--> restore index after deletion");
RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("url-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").execute().actionGet();
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
logger.info("--> list available shapshots");
GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
logger.info("--> delete snapshot");
DeleteSnapshotResponse deleteSnapshotResponse = client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get();
assertAcked(deleteSnapshotResponse);
logger.info("--> list available shapshot again, no snapshots should be returned");
getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0));
}
@TestLogging("_root:DEBUG") // this fails every now and then: https://github.com/elastic/elasticsearch/issues/18121 but without
// more logs we cannot find out why
public void testReadonlyRepository() throws Exception {

29
modules/repository-url/build.gradle Normal file
View File

@ -0,0 +1,29 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
esplugin {
description 'Module for URL repository'
classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin'
}
integTest {
cluster {
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
}
}

0
core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java → modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java
View File

3
core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java → modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java
View File

@ -55,7 +55,8 @@ public class URLBlobStore extends AbstractComponent implements BlobStore {
public URLBlobStore(Settings settings, URL path) {
super(settings);
this.path = path;
this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes();
this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size",
new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes();
}
/**

50
modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.plugin.repository.url;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.env.Environment;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.RepositoryPlugin;
import org.elasticsearch.repositories.Repository;
import org.elasticsearch.repositories.url.URLRepository;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
public class URLRepositoryPlugin extends Plugin implements RepositoryPlugin {
@Override
public List<Setting<?>> getSettings() {
return Arrays.asList(
URLRepository.ALLOWED_URLS_SETTING,
URLRepository.REPOSITORIES_URL_SETTING,
URLRepository.SUPPORTED_PROTOCOLS_SETTING
);
}
@Override
public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
return Collections.singletonMap(URLRepository.TYPE, metadata -> new URLRepository(metadata, env, namedXContentRegistry));
}
}

10
core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java → modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.elasticsearch.repositories.uri;
package org.elasticsearch.repositories.url;
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
import org.elasticsearch.common.blobstore.BlobPath;
@ -127,8 +127,12 @@ public class URLRepository extends BlobStoreRepository {
// We didn't match white list - try to resolve against path.repo
URL normalizedUrl = environment.resolveRepoURL(url);
if (normalizedUrl == null) {
logger.warn("The specified url [{}] doesn't start with any repository paths specified by the path.repo setting or by {} setting: [{}] ", url, ALLOWED_URLS_SETTING.getKey(), environment.repoFiles());
throw new RepositoryException(getMetadata().name(), "file url [" + url + "] doesn't match any of the locations specified by path.repo or " + ALLOWED_URLS_SETTING.getKey());
String logMessage = "The specified url [{}] doesn't start with any repository paths specified by the " +
"path.repo setting or by {} setting: [{}] ";
logger.warn(logMessage, url, ALLOWED_URLS_SETTING.getKey(), environment.repoFiles());
String exceptionMessage = "file url [" + url + "] doesn't match any of the locations specified by path.repo or "
+ ALLOWED_URLS_SETTING.getKey();
throw new RepositoryException(getMetadata().name(), exceptionMessage);
}
return normalizedUrl;
}

102
modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java Normal file
View File

@ -0,0 +1,102 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.blobstore.url;
import com.sun.net.httpserver.HttpServer;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.mocksocket.MockHttpServer;
import org.elasticsearch.test.ESTestCase;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.NoSuchFileException;
@SuppressForbidden(reason = "use http server")
public class URLBlobStoreTests extends ESTestCase {
private static HttpServer httpServer;
private static String blobName;
private static byte[] message = new byte[512];
private URLBlobStore urlBlobStore;
@BeforeClass
public static void startHttp() throws Exception {
for (int i = 0; i < message.length; ++i) {
message[i] = randomByte();
}
blobName = randomAsciiOfLength(8);
httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 6001), 0);
httpServer.createContext("/indices/" + blobName, (s) -> {
s.sendResponseHeaders(200, message.length);
OutputStream responseBody = s.getResponseBody();
responseBody.write(message);
responseBody.close();
});
httpServer.start();
}
@AfterClass
public static void stopHttp() throws IOException {
httpServer.stop(0);
httpServer = null;
}
@Before
public void storeSetup() throws MalformedURLException {
Settings settings = Settings.EMPTY;
String spec = "http://localhost:6001/";
urlBlobStore = new URLBlobStore(settings, new URL(spec));
}
public void testURLBlobStoreCanReadBlob() throws IOException {
BlobContainer container = urlBlobStore.blobContainer(BlobPath.cleanPath().add("indices"));
try (InputStream stream = container.readBlob(blobName)) {
byte[] bytes = new byte[message.length];
int read = stream.read(bytes);
assertEquals(message.length, read);
assertArrayEquals(message, bytes);
}
}
public void testNoBlobFound() throws IOException {
BlobContainer container = urlBlobStore.blobContainer(BlobPath.cleanPath().add("indices"));
String incorrectBlobName = "incorrect_" + blobName;
try (InputStream ignored = container.readBlob(incorrectBlobName)) {
fail("Should have thrown NoSuchFileException exception");
ignored.read();
} catch (NoSuchFileException e) {
assertEquals(String.format("[%s] blob not found", incorrectBlobName), e.getMessage());
}
}
}

40
modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.url;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
import java.io.IOException;
public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
public RepositoryURLClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}
@ParametersFactory
public static Iterable<Object[]> parameters() throws IOException {
return ESClientYamlSuiteTestCase.createParameters();
}
}

81
modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.url;
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.env.Environment;
import org.elasticsearch.repositories.RepositoryException;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Collections;
public class URLRepositoryTests extends ESTestCase {
public void testWhiteListingRepoURL() throws IOException {
String repoPath = createTempDir().resolve("repository").toUri().toURL().toString();
Settings baseSettings = Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.put(URLRepository.ALLOWED_URLS_SETTING.getKey(), repoPath)
.put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath)
.build();
RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList()));
}
public void testIfNotWhiteListedMustSetRepoURL() throws IOException {
String repoPath = createTempDir().resolve("repository").toUri().toURL().toString();
Settings baseSettings = Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath)
.build();
RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
try {
new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList()));
fail("RepositoryException should have been thrown.");
} catch (RepositoryException e) {
String msg = "[url] file url [" + repoPath
+ "] doesn't match any of the locations specified by path.repo or repositories.url.allowed_urls";
assertEquals(msg, e.getMessage());
}
}
public void testMustBeSupportedProtocol() throws IOException {
Path directory = createTempDir();
String repoPath = directory.resolve("repository").toUri().toURL().toString();
Settings baseSettings = Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.put(Environment.PATH_REPO_SETTING.getKey(), directory.toString())
.put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath)
.put(URLRepository.SUPPORTED_PROTOCOLS_SETTING.getKey(), "http,https")
.build();
RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
try {
new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList()));
fail("RepositoryException should have been thrown.");
} catch (RepositoryException e) {
assertEquals("[url] unsupported url protocol [file] from URL [" + repoPath +"]", e.getMessage());
}
}
}

131
modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java Normal file
View File

@ -0,0 +1,131 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.url;
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse;
import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.plugin.repository.url.URLRepositoryPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.snapshots.SnapshotState;
import org.elasticsearch.test.ESIntegTestCase;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Collections;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.notNullValue;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
public class URLSnapshotRestoreTests extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Collections.singletonList(URLRepositoryPlugin.class);
}
public void testUrlRepository() throws Exception {
Client client = client();
logger.info("--> creating repository");
Path repositoryLocation = randomRepoPath();
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
.setType(FsRepository.TYPE).setSettings(Settings.builder()
.put(FsRepository.LOCATION_SETTING.getKey(), repositoryLocation)
.put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
.put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
createIndex("test-idx");
ensureGreen();
logger.info("--> indexing some data");
for (int i = 0; i < 100; i++) {
index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
}
refresh();
assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
logger.info("--> snapshot");
CreateSnapshotResponse createSnapshotResponse = client
.admin()
.cluster()
.prepareCreateSnapshot("test-repo", "test-snap")
.setWaitForCompletion(true)
.setIndices("test-idx")
.get();
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
int actualTotalShards = createSnapshotResponse.getSnapshotInfo().totalShards();
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(actualTotalShards));
SnapshotState state = client
.admin()
.cluster()
.prepareGetSnapshots("test-repo")
.setSnapshots("test-snap")
.get()
.getSnapshots()
.get(0)
.state();
assertThat(state, equalTo(SnapshotState.SUCCESS));
logger.info("--> delete index");
cluster().wipeIndices("test-idx");
logger.info("--> create read-only URL repository");
assertAcked(client.admin().cluster().preparePutRepository("url-repo")
.setType(URLRepository.TYPE).setSettings(Settings.builder()
.put(URLRepository.URL_SETTING.getKey(), repositoryLocation.toUri().toURL())
.put("list_directories", randomBoolean())));
logger.info("--> restore index after deletion");
RestoreSnapshotResponse restoreSnapshotResponse = client
.admin()
.cluster()
.prepareRestoreSnapshot("url-repo", "test-snap")
.setWaitForCompletion(true)
.setIndices("test-idx")
.execute()
.actionGet();
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
logger.info("--> list available shapshots");
GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
logger.info("--> delete snapshot");
DeleteSnapshotResponse deleteSnapshotResponse = client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get();
assertAcked(deleteSnapshotResponse);
logger.info("--> list available shapshot again, no snapshots should be returned");
getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0));
}
}

32
modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yaml Normal file
View File

@ -0,0 +1,32 @@
# Integration tests for URL Repository component
#
"URL Repository plugin loaded":
- do:
cluster.state: {}
# Get master node id
- set: { master_node: master }
- do:
nodes.info: {}
- match: { nodes.$master.modules.0.name: repository-url }
---
setup:
- do:
snapshot.create_repository:
repository: test_repo1
body:
type: url
settings:
url: "http://snapshot.test1"
- do:
snapshot.create_repository:
repository: test_repo2
body:
type: url
settings:
url: "http://snapshot.test2"

16
modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yaml Normal file
View File

@ -0,0 +1,16 @@
"Repository can be registered":
- do:
snapshot.create_repository:
repository: test_repo1
body:
type: url
settings:
url: "http://snapshot.test1"
- is_true: acknowledged
- do:
snapshot.get_repository:
repository: test_repo1
- is_true : test_repo1

3
qa/rolling-upgrade/build.gradle
View File

@ -29,6 +29,7 @@ task oldClusterTest(type: RestIntegTestTask) {
numBwcNodes = 2
numNodes = 2
clusterName = 'rolling-upgrade'
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
}
systemProperty 'tests.rest.suite', 'old_cluster'
}
@ -40,6 +41,7 @@ task mixedClusterTest(type: RestIntegTestTask) {
clusterName = 'rolling-upgrade'
unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
}
systemProperty 'tests.rest.suite', 'mixed_cluster'
finalizedBy 'oldClusterTest#node0.stop'
@ -52,6 +54,7 @@ task upgradedClusterTest(type: RestIntegTestTask) {
clusterName = 'rolling-upgrade'
unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
dataDir = "${-> oldClusterTest.nodes[0].dataDir}"
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
}
systemProperty 'tests.rest.suite', 'upgraded_cluster'
// only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion

8
rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/10_basic.yaml
View File

@ -5,17 +5,17 @@ setup:
snapshot.create_repository:
repository: test_repo_get_1
body:
type: url
type: fs
settings:
url: "http://snapshot.test1"
location: "test_repo_get_1_loc"
- do:
snapshot.create_repository:
repository: test_repo_get_2
body:
type: url
type: fs
settings:
url: "http://snapshot.test2"
location: "test_repo_get_1_loc"
---
"Get all repositories":

1
settings.gradle
View File

@ -31,6 +31,7 @@ List projects = [
'modules:transport-netty4',
'modules:reindex',
'modules:percolator',
'modules:repository-url',
'plugins:analysis-icu',
'plugins:analysis-kuromoji',
'plugins:analysis-phonetic',