diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 8861d97a1ae..8dc504afd78 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -149,7 +149,8 @@ Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this r New Features --------------------- -(No changes) + +* SOLR-14907: Add v2 API for configSet upload, including single-file insertion. (Houston Putman) Improvements --------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java index 81ab2b72c79..f3f28161b4e 100644 --- a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java @@ -47,6 +47,7 @@ import org.apache.solr.response.SolrQueryResponse; import static org.apache.solr.client.solrj.SolrRequest.METHOD.DELETE; import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET; import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST; +import static org.apache.solr.client.solrj.SolrRequest.METHOD.PUT; import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.REQUESTID; import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE; import static org.apache.solr.common.params.CollectionParams.CollectionAction.CLUSTERPROP; @@ -133,6 +134,37 @@ public class ClusterAPI { } + @EndPoint(method = PUT, + path = "/cluster/configs/{name}", + permission = CONFIG_EDIT_PERM + ) + public void uploadConfigSet(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { + req = wrapParams(req, + "action", ConfigSetParams.ConfigSetAction.UPLOAD.toString(), + CommonParams.NAME, req.getPathTemplateValues().get("name"), + ConfigSetParams.OVERWRITE, true, + ConfigSetParams.CLEANUP, false); + configSetsHandler.handleRequestBody(req, rsp); + } + + @EndPoint(method = PUT, + path = "/cluster/configs/{name}/*", + permission = CONFIG_EDIT_PERM + ) + public void insertIntoConfigSet(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { + String path = req.getPathTemplateValues().get("*"); + if (path == null || path.isBlank()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "In order to insert a file in a configSet, a filePath must be provided in the url after the name of the configSet."); + } + req = wrapParams(req, + "action", ConfigSetParams.ConfigSetAction.UPLOAD.toString(), + CommonParams.NAME, req.getPathTemplateValues().get("name"), + ConfigSetParams.FILE_PATH, path, + ConfigSetParams.OVERWRITE, true, + ConfigSetParams.CLEANUP, false); + configSetsHandler.handleRequestBody(req, rsp); + } + @SuppressWarnings({"rawtypes"}) public static SolrQueryRequest wrapParams(SolrQueryRequest req, Object... def) { Map m = Utils.makeMap(def); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java index 736aeb7b56e..cb6d2a3de85 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java @@ -68,6 +68,7 @@ import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.CREATE; import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.DELETE; import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.LIST; +import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.UPLOAD; /** * A {@link org.apache.solr.request.SolrRequestHandler} for ConfigSets API requests. @@ -193,10 +194,10 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN fixedSingleFilePath = fixedSingleFilePath.substring(1); } if (fixedSingleFilePath.isEmpty()) { - throw new SolrException(ErrorCode.BAD_REQUEST, "The filePath provided for upload, '" + singleFilePath + "', is not valid."); + throw new SolrException(ErrorCode.BAD_REQUEST, "The file path provided for upload, '" + singleFilePath + "', is not valid."); } else if (cleanup) { // Cleanup is not allowed while using singleFilePath upload - throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet uploads do not allow cleanup=true when filePath is used."); + throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet uploads do not allow cleanup=true when file path is used."); } else { try { // Create a node for the configuration in zookeeper @@ -206,7 +207,7 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN zkClient.makePath(filePathInZk, IOUtils.toByteArray(inputStream), CreateMode.PERSISTENT, null, !allowOverwrite, true); } catch(KeeperException.NodeExistsException nodeExistsException) { throw new SolrException(ErrorCode.BAD_REQUEST, - "The path " + singleFilePath + " for configSet " + configSetName + " already exists. In order to overwrite, provide overwrite=true."); + "The path " + singleFilePath + " for configSet " + configSetName + " already exists. In order to overwrite, provide overwrite=true or use an HTTP PUT with the V2 API."); } } return; @@ -230,7 +231,9 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN ZipInputStream zis = new ZipInputStream(inputStream, StandardCharsets.UTF_8); ZipEntry zipEntry = null; + boolean hasEntry = false; while ((zipEntry = zis.getNextEntry()) != null) { + hasEntry = true; String filePathInZk = configPathInZk + "/" + zipEntry.getName(); if (filePathInZk.endsWith("/")) { filesToDelete.remove(filePathInZk.substring(0, filePathInZk.length() -1)); @@ -245,6 +248,10 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN } } zis.close(); + if (!hasEntry) { + throw new SolrException(ErrorCode.BAD_REQUEST, + "Either empty zipped data, or non-zipped data was uploaded. In order to upload a configSet, you must zip a non-empty directory to upload."); + } deleteUnusedFiles(zkClient, filesToDelete); // If the request is doing a full trusted overwrite of an untrusted configSet (overwrite=true, cleanup=true), then trust the configSet. @@ -400,6 +407,13 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN } public enum ConfigSetOperation { + UPLOAD_OP(UPLOAD) { + @Override + public Map call(SolrQueryRequest req, SolrQueryResponse rsp, ConfigSetsHandler h) throws Exception { + h.handleConfigUploadRequest(req, rsp); + return null; + } + }, CREATE_OP(CREATE) { @Override public Map call(SolrQueryRequest req, SolrQueryResponse rsp, ConfigSetsHandler h) throws Exception { diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java index c7b0a82466c..65b09d56e7d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java @@ -52,7 +52,9 @@ import com.google.common.collect.ImmutableMap; import org.apache.commons.io.FileUtils; import org.apache.http.HttpEntity; import org.apache.http.auth.BasicUserPrincipal; +import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.message.BasicHeader; import org.apache.http.util.EntityUtils; @@ -343,7 +345,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { @SuppressWarnings({"rawtypes"}) Map map = postDataAndGetResponse(cluster.getSolrClient(), cluster.getJettySolrRunners().get(0).getBaseUrl().toString() - + "/admin/configs?action=UPLOAD", emptyData, null); + + "/admin/configs?action=UPLOAD", emptyData, null, false); assertNotNull(map); unIgnoreException("The configuration name should be provided"); long statusCode = (long) getObjectByPath(map, false, @@ -364,7 +366,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { ignoreException("already exists"); map = postDataAndGetResponse(cluster.getSolrClient(), cluster.getJettySolrRunners().get(0).getBaseUrl().toString() - + "/admin/configs?action=UPLOAD&name=myconf", emptyData, null); + + "/admin/configs?action=UPLOAD&name=myconf", emptyData, null, false); assertNotNull(map); unIgnoreException("already exists`"); statusCode = (long) getObjectByPath(map, false, @@ -380,7 +382,16 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { } @Test - public void testUploadDisabled() throws Exception { + public void testUploadDisabledV1() throws Exception { + testUploadDisabled(false); + } + + @Test + public void testUploadDisabledV2() throws Exception { + testUploadDisabled(true); + } + + public void testUploadDisabled(boolean v2) throws Exception { try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null)) { @@ -388,7 +399,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { for (boolean enabled: new boolean[] {true, false}) { System.setProperty("configset.upload.enabled", String.valueOf(enabled)); try { - long statusCode = uploadConfigSet("regular", "test-enabled-is-" + enabled, null, zkClient); + long statusCode = uploadConfigSet("regular", "test-enabled-is-" + enabled, null, zkClient, v2); assertEquals("ConfigSet upload enabling/disabling not working as expected for enabled=" + enabled + ".", enabled? 0l: 400l, statusCode); } finally { @@ -400,20 +411,29 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { } @Test - public void testOverwrite() throws Exception { + public void testOverwriteV1() throws Exception { + testOverwrite(false); + } + + @Test + public void testOverwriteV2() throws Exception { + testOverwrite(true); + } + + public void testOverwrite(boolean v2) throws Exception { String configsetName = "regular"; - String configsetSuffix = "testOverwrite-1"; + String configsetSuffix = "testOverwrite-1-" + v2; uploadConfigSetWithAssertions(configsetName, configsetSuffix, null); try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null)) { int solrconfigZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"); ignoreException("The configuration regulartestOverwrite-1 already exists in zookeeper"); assertEquals("Can't overwrite an existing configset unless the overwrite parameter is set", - 400, uploadConfigSet(configsetName, configsetSuffix, null, zkClient, false, false)); + 400, uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2)); unIgnoreException("The configuration regulartestOverwrite-1 already exists in zookeeper"); assertEquals("Expecting version to remain equal", solrconfigZkVersion, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml")); - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, zkClient, true, false)); + assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2)); assertTrue("Expecting version bump", solrconfigZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml")); } @@ -421,25 +441,35 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { } @Test - public void testOverwriteWithCleanup() throws Exception { + public void testOverwriteWithCleanupV1() throws Exception { + testOverwriteWithCleanup(false); + } + + @Test + public void testOverwriteWithCleanupV2() throws Exception { + testOverwriteWithCleanup(true); + } + + public void testOverwriteWithCleanup(boolean v2) throws Exception { String configsetName = "regular"; - String configsetSuffix = "testOverwriteWithCleanup-1"; + String configsetSuffix = "testOverwriteWithCleanup-1-" + v2; uploadConfigSetWithAssertions(configsetName, configsetSuffix, null); try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null)) { + String configPath = "/configs/" + configsetName + configsetSuffix; List extraFiles = Arrays.asList( - "/configs/regulartestOverwriteWithCleanup-1/foo1", - "/configs/regulartestOverwriteWithCleanup-1/foo2", - "/configs/regulartestOverwriteWithCleanup-1/foo2/1", - "/configs/regulartestOverwriteWithCleanup-1/foo2/2"); + configPath + "/foo1", + configPath + "/foo2", + configPath + "/foo2/1", + configPath + "/foo2/2"); for (String f : extraFiles) { zkClient.makePath(f, true); } - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, zkClient, true, false)); + assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2)); for (String f : extraFiles) { assertTrue("Expecting file " + f + " to exist in ConfigSet but it's gone", zkClient.exists(f, true)); } - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, zkClient, true, true)); + assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2)); for (String f : extraFiles) { assertFalse("Expecting file " + f + " to be deleted from ConfigSet but it wasn't", zkClient.exists(f, true)); } @@ -448,38 +478,49 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { } @Test - public void testOverwriteWithTrust() throws Exception { + public void testOverwriteWithTrustV1() throws Exception { + testOverwriteWithTrust(false); + } + + @Test + public void testOverwriteWithTrustV2() throws Exception { + testOverwriteWithTrust(true); + } + + public void testOverwriteWithTrust(boolean v2) throws Exception { String configsetName = "regular"; - String configsetSuffix = "testOverwriteWithTrust-1"; + String configsetSuffix = "testOverwriteWithTrust-1-" + v2; uploadConfigSetWithAssertions(configsetName, configsetSuffix, null); try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null)) { assertFalse(isTrusted(zkClient, configsetName, configsetSuffix)); int solrconfigZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"); // Was untrusted, overwrite with untrusted - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, zkClient, true, false)); + assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2)); assertTrue("Expecting version bump", solrconfigZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml")); assertFalse(isTrusted(zkClient, configsetName, configsetSuffix)); solrconfigZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"); // Was untrusted, overwrite with trusted but no cleanup - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", zkClient, true, false)); + assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2)); assertTrue("Expecting version bump", solrconfigZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml")); assertFalse(isTrusted(zkClient, configsetName, configsetSuffix)); solrconfigZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"); // Was untrusted, overwrite with trusted with cleanup but fail on unzipping. - // Should not set trusted=true - assertEquals(500, uploadBadConfigSet(configsetName, configsetSuffix, "solr", zkClient, true, true)); + // Should not set trusted=true in configSet + ignoreException("Either empty zipped data, or non-zipped data was passed. In order to upload a configSet, you must zip a non-empty directory to upload."); + assertEquals(400, uploadBadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2)); assertEquals("Expecting version bump", solrconfigZkVersion, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml")); assertFalse(isTrusted(zkClient, configsetName, configsetSuffix)); solrconfigZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"); + ignoreException("Either empty zipped data, or non-zipped data was passed. In order to upload a configSet, you must zip a non-empty directory to upload."); // Was untrusted, overwrite with trusted with cleanup - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", zkClient, true, true)); + assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2)); assertTrue("Expecting version bump", solrconfigZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml")); assertTrue(isTrusted(zkClient, configsetName, configsetSuffix)); @@ -488,7 +529,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { // Was trusted, try to overwrite with untrusted with no cleanup ignoreException("Trying to make an unstrusted ConfigSet update on a trusted configSet"); assertEquals("Can't upload a trusted configset with an untrusted request", - 400, uploadConfigSet(configsetName, configsetSuffix, null, zkClient, true, false)); + 400, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2)); assertEquals("Expecting version to remain equal", solrconfigZkVersion, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml")); assertTrue(isTrusted(zkClient, configsetName, configsetSuffix)); @@ -496,21 +537,21 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { // Was trusted, try to overwrite with untrusted with cleanup ignoreException("Trying to make an unstrusted ConfigSet update on a trusted configSet"); assertEquals("Can't upload a trusted configset with an untrusted request", - 400, uploadConfigSet(configsetName, configsetSuffix, null, zkClient, true, true)); + 400, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2)); assertEquals("Expecting version to remain equal", solrconfigZkVersion, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml")); assertTrue(isTrusted(zkClient, configsetName, configsetSuffix)); unIgnoreException("Trying to make an unstrusted ConfigSet update on a trusted configSet"); // Was trusted, overwrite with trusted no cleanup - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", zkClient, true, false)); + assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2)); assertTrue("Expecting version bump", solrconfigZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml")); assertTrue(isTrusted(zkClient, configsetName, configsetSuffix)); solrconfigZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"); // Was trusted, overwrite with trusted with cleanup - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", zkClient, true, true)); + assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2)); assertTrue("Expecting version bump", solrconfigZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml")); assertTrue(isTrusted(zkClient, configsetName, configsetSuffix)); @@ -519,47 +560,74 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { } @Test - public void testSingleFileOverwrite() throws Exception { + public void testSingleFileOverwriteV1() throws Exception { + testSingleFileOverwrite(false); + } + + @Test + public void testSingleFileOverwriteV2() throws Exception { + testSingleFileOverwrite(true); + } + + public void testSingleFileOverwrite(boolean v2) throws Exception { String configsetName = "regular"; - String configsetSuffix = "testSinglePathOverwrite-1"; + String configsetSuffix = "testSinglePathOverwrite-1-" + v2; uploadConfigSetWithAssertions(configsetName, configsetSuffix, null); try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null)) { int solrconfigZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"); ignoreException("The configuration regulartestOverwrite-1 already exists in zookeeper"); assertEquals("Can't overwrite an existing configset unless the overwrite parameter is set", - 400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", false, false)); + 400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", false, false, v2)); unIgnoreException("The configuration regulartestOverwrite-1 already exists in zookeeper"); assertEquals("Expecting version to remain equal", solrconfigZkVersion, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml")); - assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", true, false)); + assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", true, false, v2)); assertTrue("Expecting version bump", solrconfigZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml")); } } @Test - public void testNewSingleFile() throws Exception { + public void testNewSingleFileV1() throws Exception { + testNewSingleFile(false); + } + + @Test + public void testNewSingleFileV2() throws Exception { + testNewSingleFile(true); + } + + public void testNewSingleFile(boolean v2) throws Exception { String configsetName = "regular"; - String configsetSuffix = "testSinglePathNew-1"; + String configsetSuffix = "testSinglePathNew-1-" + v2; uploadConfigSetWithAssertions(configsetName, configsetSuffix, null); try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null)) { - assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", false, false)); + assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", false, false, v2)); assertEquals("Expecting first version of new file", 0, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/upload/path/solrconfig.xml")); assertConfigsetFiles(configsetName, configsetSuffix, zkClient); } } @Test - public void testSingleWithCleanup() throws Exception { + public void testSingleWithCleanupV1() throws Exception { + testSingleWithCleanup(false); + } + + @Test + public void testSingleWithCleanupV2() throws Exception { + testSingleWithCleanup(true); + } + + public void testSingleWithCleanup(boolean v2) throws Exception { String configsetName = "regular"; - String configsetSuffix = "testSinglePathCleanup-1"; + String configsetSuffix = "testSinglePathCleanup-1-" + v2; uploadConfigSetWithAssertions(configsetName, configsetSuffix, null); try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null)) { ignoreException("ConfigSet uploads do not allow cleanup=true when filePath is used."); - assertEquals(400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", false, true)); + assertEquals(400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", true, true, v2)); assertFalse("New file should not exist, since the trust check did not succeed.", zkClient.exists("/configs/"+configsetName+configsetSuffix+"/test/upload/path/solrconfig.xml", true)); assertConfigsetFiles(configsetName, configsetSuffix, zkClient); unIgnoreException("ConfigSet uploads do not allow cleanup=true when filePath is used."); @@ -567,20 +635,29 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { } @Test - public void testSingleFileTrusted() throws Exception { + public void testSingleFileTrustedV1() throws Exception { + testSingleFileTrusted(false); + } + + @Test + public void testSingleFileTrustedV2() throws Exception { + testSingleFileTrusted(true); + } + + public void testSingleFileTrusted(boolean v2) throws Exception { String configsetName = "regular"; - String configsetSuffix = "testSinglePathTrusted-1"; + String configsetSuffix = "testSinglePathTrusted-1-" + v2; uploadConfigSetWithAssertions(configsetName, configsetSuffix, "solr"); try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null)) { - assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", true, false)); + assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", true, false, v2)); assertEquals("Expecting first version of new file", 0, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/upload/path/solrconfig.xml")); assertTrue(isTrusted(zkClient, configsetName, configsetSuffix)); assertConfigsetFiles(configsetName, configsetSuffix, zkClient); ignoreException("Trying to make an unstrusted ConfigSet update on a trusted configSet"); assertEquals("Can't upload a trusted configset with an untrusted request", - 400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", true, false)); + 400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", true, false, v2)); assertFalse("New file should not exist, since the trust check did not succeed.", zkClient.exists("/configs/"+configsetName+configsetSuffix+"/test/different/path/solrconfig.xml", true)); assertTrue(isTrusted(zkClient, configsetName, configsetSuffix)); assertConfigsetFiles(configsetName, configsetSuffix, zkClient); @@ -589,7 +666,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { ignoreException("Trying to make an unstrusted ConfigSet update on a trusted configSet"); int extraFileZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/upload/path/solrconfig.xml"); assertEquals("Can't upload a trusted configset with an untrusted request", - 400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", true, false)); + 400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", true, false, v2)); assertEquals("Expecting version to remain equal", extraFileZkVersion, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/upload/path/solrconfig.xml")); assertTrue(isTrusted(zkClient, configsetName, configsetSuffix)); @@ -599,27 +676,36 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { } @Test - public void testSingleFileUntrusted() throws Exception { + public void testSingleFileUntrustedV1() throws Exception { + testSingleFileUntrusted(false); + } + + @Test + public void testSingleFileUntrustedV2() throws Exception { + testSingleFileUntrusted(true); + } + + public void testSingleFileUntrusted(boolean v2) throws Exception { String configsetName = "regular"; - String configsetSuffix = "testSinglePathUntrusted-1"; + String configsetSuffix = "testSinglePathUntrusted-1-" + v2; uploadConfigSetWithAssertions(configsetName, configsetSuffix, null); try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null)) { // New file with trusted request - assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", false, false)); + assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", false, false, v2)); assertEquals("Expecting first version of new file", 0, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/upload/path/solrconfig.xml")); assertFalse(isTrusted(zkClient, configsetName, configsetSuffix)); assertConfigsetFiles(configsetName, configsetSuffix, zkClient); // New file with untrusted request - assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", false, false)); + assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", false, false, v2)); assertEquals("Expecting first version of new file", 0, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/different/path/solrconfig.xml")); assertFalse(isTrusted(zkClient, configsetName, configsetSuffix)); assertConfigsetFiles(configsetName, configsetSuffix, zkClient); // Overwrite with trusted request int extraFileZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/different/path/solrconfig.xml"); - assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", true, false)); + assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", true, false, v2)); assertTrue("Expecting version bump", extraFileZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/different/path/solrconfig.xml")); assertFalse(isTrusted(zkClient, configsetName, configsetSuffix)); @@ -627,7 +713,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { // Overwrite with untrusted request extraFileZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/upload/path/solrconfig.xml"); - assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", true, false)); + assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", true, false, v2)); assertTrue("Expecting version bump", extraFileZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/upload/path/solrconfig.xml")); assertFalse(isTrusted(zkClient, configsetName, configsetSuffix)); @@ -636,7 +722,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { // Make sure that cleanup flag does not result in configSet being trusted. ignoreException("ConfigSet uploads do not allow cleanup=true when filePath is used."); extraFileZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/different/path/solrconfig.xml"); - assertEquals(400, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", true, true)); + assertEquals(400, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", true, true, v2)); assertEquals("Expecting version to stay the same", extraFileZkVersion, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/different/path/solrconfig.xml")); assertFalse("The cleanup=true flag allowed for trust overwriting in a filePath upload.", isTrusted(zkClient, configsetName, configsetSuffix)); @@ -646,14 +732,23 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { } @Test - public void testSingleFileNewConfig() throws Exception { + public void testSingleFileNewConfigV1() throws Exception { + testSingleFileNewConfig(false); + } + + @Test + public void testSingleFileNewConfigV2() throws Exception { + testSingleFileNewConfig(true); + } + + public void testSingleFileNewConfig(boolean v2) throws Exception { String configsetName = "regular"; - String configsetSuffixTrusted = "testSinglePathNewConfig-1"; - String configsetSuffixUntrusted = "testSinglePathNewConfig-2"; + String configsetSuffixTrusted = "testSinglePathNewConfig-1-" + v2; + String configsetSuffixUntrusted = "testSinglePathNewConfig-2-" + v2; try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null)) { // New file with trusted request - assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffixTrusted, "solr", zkClient, "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", false, false)); + assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffixTrusted, "solr", "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", false, false, v2)); assertEquals("Expecting first version of new file", 0, getConfigZNodeVersion(zkClient, configsetName, configsetSuffixTrusted, "solrconfig.xml")); assertTrue(isTrusted(zkClient, configsetName, configsetSuffixTrusted)); List children = zkClient.getChildren(String.format(Locale.ROOT,"/configs/%s%s", configsetName, configsetSuffixTrusted), null, true); @@ -661,7 +756,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { assertEquals("Incorrect file uploaded.", "solrconfig.xml", children.get(0)); // New file with trusted request - assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffixUntrusted, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", false, false)); + assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffixUntrusted, null, "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", false, false, v2)); assertEquals("Expecting first version of new file", 0, getConfigZNodeVersion(zkClient, configsetName, configsetSuffixUntrusted, "solrconfig.xml")); assertFalse(isTrusted(zkClient, configsetName, configsetSuffixUntrusted)); children = zkClient.getChildren(String.format(Locale.ROOT,"/configs/%s%s", configsetName, configsetSuffixUntrusted), null, true); @@ -762,7 +857,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null); try { - long statusCode = uploadConfigSet(configSetName, suffix, username, zkClient); + long statusCode = uploadConfigSet(configSetName, suffix, username, zkClient, true); assertEquals(0l, statusCode); assertConfigsetFiles(configSetName, suffix, zkClient); } finally { @@ -786,51 +881,68 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { } private long uploadConfigSet(String configSetName, String suffix, String username, - SolrZkClient zkClient) throws IOException { + SolrZkClient zkClient, boolean v2) throws IOException { ZkConfigManager configManager = new ZkConfigManager(zkClient); assertFalse(configManager.configExists(configSetName + suffix)); - return uploadConfigSet(configSetName, suffix, username, zkClient, false, false); + return uploadConfigSet(configSetName, suffix, username, false, false, v2); } private long uploadConfigSet(String configSetName, String suffix, String username, - SolrZkClient zkClient, boolean overwrite, boolean cleanup) throws IOException { + boolean overwrite, boolean cleanup, boolean v2) throws IOException { // Read zipped sample config ByteBuffer sampleZippedConfig = TestSolrConfigHandler .getFileContent( createTempZipFile("solr/configsets/upload/"+configSetName), false); - @SuppressWarnings({"rawtypes"}) - Map map = postDataAndGetResponse(cluster.getSolrClient(), - cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/admin/configs?action=UPLOAD&name="+configSetName+suffix + (overwrite? "&overwrite=true" : "") + (cleanup? "&cleanup=true" : ""), - sampleZippedConfig, username); - assertNotNull(map); - long statusCode = (long) getObjectByPath(map, false, Arrays.asList("responseHeader", "status")); - return statusCode; + return uploadGivenConfigSet(sampleZippedConfig, configSetName, suffix, username, overwrite, cleanup, v2); } private long uploadBadConfigSet(String configSetName, String suffix, String username, - SolrZkClient zkClient, boolean overwrite, boolean cleanup) throws IOException { + boolean overwrite, boolean cleanup, boolean v2) throws IOException { // Read single file from sample configs. This should fail the unzipping ByteBuffer sampleBadZippedFile = TestSolrConfigHandler.getFileContent(SolrTestCaseJ4.getFile("solr/configsets/upload/regular/solrconfig.xml").getAbsolutePath(), false); + return uploadGivenConfigSet(sampleBadZippedFile, configSetName, suffix, username, overwrite, cleanup, v2); + } + + private long uploadGivenConfigSet(ByteBuffer file, String configSetName, String suffix, String username, + boolean overwrite, boolean cleanup, boolean v2) throws IOException { + String uriEnding; + boolean usePut = false; + if (v2) { + uriEnding = "/api/cluster/configs/" + configSetName+suffix + (!overwrite? "?overwrite=false" : "") + (cleanup? "?cleanup=true" : ""); + usePut = true; + } else { + uriEnding = "/solr/admin/configs?action=UPLOAD&name="+configSetName+suffix + (overwrite? "&overwrite=true" : "") + (cleanup? "&cleanup=true" : ""); + } + @SuppressWarnings({"rawtypes"}) Map map = postDataAndGetResponse(cluster.getSolrClient(), - cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/admin/configs?action=UPLOAD&name="+configSetName+suffix + (overwrite? "&overwrite=true" : "") + (cleanup? "&cleanup=true" : ""), - sampleBadZippedFile, username); + cluster.getJettySolrRunners().get(0).getBaseUrl().toString().replace("/solr", "") + uriEnding, + file, username, usePut); assertNotNull(map); long statusCode = (long) getObjectByPath(map, false, Arrays.asList("responseHeader", "status")); return statusCode; } private long uploadSingleConfigSetFile(String configSetName, String suffix, String username, - SolrZkClient zkClient, String filePath, String uploadPath, boolean overwrite, boolean cleanup) throws IOException { + String filePath, String uploadPath, boolean overwrite, boolean cleanup, boolean v2) throws IOException { // Read single file from sample configs ByteBuffer sampleConfigFile = TestSolrConfigHandler.getFileContent(SolrTestCaseJ4.getFile(filePath).getAbsolutePath(), false); + String uriEnding; + boolean usePut = false; + if (v2) { + uriEnding = "/api/cluster/configs/" + configSetName+suffix + "/" + uploadPath + (!overwrite? "?overwrite=false" : "") + (cleanup? "?cleanup=true" : ""); + usePut = true; + } else { + uriEnding = "/solr/admin/configs?action=UPLOAD&name="+configSetName+suffix+"&filePath="+uploadPath + (overwrite? "&overwrite=true" : "") + (cleanup? "&cleanup=true" : ""); + } + @SuppressWarnings({"rawtypes"}) Map map = postDataAndGetResponse(cluster.getSolrClient(), - cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/admin/configs?action=UPLOAD&name="+configSetName+suffix+"&filePath="+uploadPath + (overwrite? "&overwrite=true" : "") + (cleanup? "&cleanup=true" : ""), - sampleConfigFile, username); + cluster.getJettySolrRunners().get(0).getBaseUrl().toString().replace("/solr", "") + uriEnding, + sampleConfigFile, username, usePut); assertNotNull(map); long statusCode = (long) getObjectByPath(map, false, Arrays.asList("responseHeader", "status")); return statusCode; @@ -928,24 +1040,28 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { @SuppressWarnings({"rawtypes"}) public static Map postDataAndGetResponse(CloudSolrClient cloudClient, - String uri, ByteBuffer bytarr, String username) throws IOException { - HttpPost httpPost = null; + String uri, ByteBuffer bytarr, String username, boolean usePut) throws IOException { + HttpEntityEnclosingRequestBase httpRequest = null; HttpEntity entity; String response = null; Map m = null; try { - httpPost = new HttpPost(uri); + if (usePut) { + httpRequest = new HttpPut(uri); + } else { + httpRequest = new HttpPost(uri); + } if (username != null) { - httpPost.addHeader(new BasicHeader("user", username)); + httpRequest.addHeader(new BasicHeader("user", username)); } - httpPost.setHeader("Content-Type", "application/octet-stream"); - httpPost.setEntity(new ByteArrayEntity(bytarr.array(), bytarr + httpRequest.setHeader("Content-Type", "application/octet-stream"); + httpRequest.setEntity(new ByteArrayEntity(bytarr.array(), bytarr .arrayOffset(), bytarr.limit())); log.info("Uploading configset with user {}", username); - entity = cloudClient.getLbClient().getHttpClient().execute(httpPost) + entity = cloudClient.getLbClient().getHttpClient().execute(httpRequest) .getEntity(); try { response = EntityUtils.toString(entity, UTF_8); @@ -955,7 +1071,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { throw new AssertionError(e); } } finally { - httpPost.releaseConnection(); + httpRequest.releaseConnection(); } return m; } diff --git a/solr/solr-ref-guide/src/configsets-api.adoc b/solr/solr-ref-guide/src/configsets-api.adoc index 820cb39942f..ead84f1f9f9 100644 --- a/solr/solr-ref-guide/src/configsets-api.adoc +++ b/solr/solr-ref-guide/src/configsets-api.adoc @@ -101,21 +101,29 @@ The configset to be created when the upload is complete. This parameter is requi `overwrite`:: If set to `true`, Solr will overwrite an existing configset with the same name (if false, the request will fail). If `filePath` is provided, then this option specifies whether the specified file within the configSet should be overwritten if it already exists. -Default is `false`. +Default is `false` when using the v1 API, but `true` when using the v2 API. `cleanup`:: When overwriting an existing configset (`overwrite=true`), this parameter tells Solr to delete the files in ZooKeeper that existed in the old configset but not in the one being uploaded. Default is `false`. This parameter cannot be set to true when `filePath` is used. -filePath:: +`filePath`:: This parameter allows the uploading of a single, non-zipped file to the given path under the configSet in ZooKeeper. This functionality respects the `overwrite` parameter, so a request will fail if the given file path already exists in the configSet and overwrite is set to `false`. The `cleanup` parameter cannot be set to true when `filePath` is used. -The body of the request should be a zip file that contains the configset. The zip file must be created from within the `conf` directory (i.e., `solrconfig.xml` must be the top level entry in the zip file). +If uploading an entire configSet, the body of the request should be a zip file that contains the configset. The zip file must be created from within the `conf` directory (i.e., `solrconfig.xml` must be the top level entry in the zip file). Here is an example on how to create the zip file named "myconfig.zip" and upload it as a configset named "myConfigSet": +[.dynamic-tabs] +-- +[example.tab-pane#v1uploadconfigset] +==== +[.tab-label]*V1 API* + +With the v1 API, the `upload` command must be capitalized as `UPLOAD`: + [source,bash] ---- $ (cd solr/server/solr/configsets/sample_techproducts_configs/conf && zip -r - *) > myconfigset.zip @@ -129,8 +137,65 @@ The same can be achieved using a Unix pipe with a single request as follows: ---- $ (cd server/solr/configsets/sample_techproducts_configs/conf && zip -r - *) | curl -X POST --header "Content-Type:application/octet-stream" --data-binary @- "http://localhost:8983/solr/admin/configs?action=UPLOAD&name=myConfigSet" ---- +==== -NOTE: The `UPLOAD` command does not yet have a v2 equivalent API. +[example.tab-pane#v2uploadconfigset] +==== +[.tab-label]*V2 API* + +With the v2 API, the name of the configset to upload is provided as a path parameter: + +[source,bash] +---- +$ (cd solr/server/solr/configsets/sample_techproducts_configs/conf && zip -r - *) > myconfigset.zip + +$ curl -X PUT --header "Content-Type:application/octet-stream" --data-binary @myconfigset.zip + "http://localhost:8983/api/cluster/configs/myConfigSet" +---- + +With this REST API, the default behavior is to overwrite the configSet if it already exists. +This behavior can be disabled by providing the URL param `overwrite=false`, in which case the request will fail if the configSet already exists. +==== +-- + +Here is an example on how to upload a single file to a configset named "myConfigSet": + +[.dynamic-tabs] +-- +[example.tab-pane#v1uploadsinglefile] +==== +[.tab-label]*V1 API* + +With the v1 API, the `upload` command must be capitalized as `UPLOAD`. +The filename to upload is provided via the `filePath` URL param: + +[source,bash] +---- +curl -X POST --header "Content-Type:application/octet-stream" + --data-binary @solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml + "http://localhost:8983/solr/admin/configs?action=UPLOAD&name=myConfigSet&filePath=solrconfig.xml&overwrite=true" +---- +==== + +[example.tab-pane#v2uploadsinglefile] +==== +[.tab-label]*V2 API* + +With the v2 API, the name of the configset and file are both provided in the URL. +They can be substituted in `/cluster/configs/{config_name}/{file_name}`. +The filename may be nested and include `/` characters. + +[source,bash] +---- +curl -X PUT --header "Content-Type:application/octet-stream" + --data-binary @solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml + "http://localhost:8983/api/cluster/configs/myConfigSet/solrconfig.xml" +---- + +With this REST API, the default behavior is to overwrite the file if it already exists within the configSet. +This behavior can be disabled by providing the URL param `overwrite=false`, in which case the request will fail if the file already exists within the configSet. +==== +-- [[configsets-create]] == Create a Configset