From 93511352ac824a1e022758600e656ea8b2892913 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Thu, 7 Apr 2016 10:44:08 +0100 Subject: [PATCH] SOLR-8938: Add optional -excluderegex argument to ZkCLI. --- solr/CHANGES.txt | 1 + .../src/java/org/apache/solr/cloud/ZkCLI.java | 13 ++++-- .../test/org/apache/solr/cloud/ZkCLITest.java | 41 ++++++++++++++----- .../solr/common/cloud/ZkConfigManager.java | 27 ++++++++++-- .../common/cloud/TestZkConfigManager.java | 7 ++++ 5 files changed, 72 insertions(+), 17 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 979389305f3..33e9a3e68a7 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -89,6 +89,7 @@ New Features graph traversal to find the shortest path(s) in an unweighted, directed graph stored in a SolrCloud collection. (Joel Bernstein) +* SOLR-8938: Add optional -excluderegex argument to ZkCLI. (Christine Poerschke) Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java index 7ba3a8fb410..402a83fdfba 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java @@ -29,7 +29,6 @@ import org.apache.commons.cli.PosixParser; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.OnReconnect; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkConfigManager; import org.apache.solr.common.cloud.ZkStateReader; @@ -48,6 +47,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.List; import java.util.concurrent.TimeoutException; +import java.util.regex.Pattern; public class ZkCLI { @@ -66,7 +66,9 @@ public class ZkCLI { private static final String RUNZK = "runzk"; private static final String SOLRHOME = "solrhome"; private static final String BOOTSTRAP = "bootstrap"; - private static final String UPCONFIG = "upconfig"; + static final String UPCONFIG = "upconfig"; + static final String EXCLUDE_REGEX = "excluderegex"; + static final String EXCLUDE_REGEX_DEFAULT = ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX; private static final String COLLECTION = "collection"; private static final String CLEAR = "clear"; private static final String LIST = "list"; @@ -120,6 +122,9 @@ public class ZkCLI { options.addOption("c", COLLECTION, true, "for " + LINKCONFIG + ": name of the collection"); + options.addOption(EXCLUDE_REGEX, true, + "for " + UPCONFIG + ": files matching this regular expression won't be uploaded"); + options .addOption( "r", @@ -211,13 +216,15 @@ public class ZkCLI { } String confDir = line.getOptionValue(CONFDIR); String confName = line.getOptionValue(CONFNAME); + final String excludeExpr = line.getOptionValue(EXCLUDE_REGEX, EXCLUDE_REGEX_DEFAULT); if(!ZkController.checkChrootPath(zkServerAddress, true)) { System.out.println("A chroot was specified in zkHost but the znode doesn't exist. "); System.exit(1); } ZkConfigManager configManager = new ZkConfigManager(zkClient); - configManager.uploadConfigDir(Paths.get(confDir), confName); + final Pattern excludePattern = Pattern.compile(excludeExpr); + configManager.uploadConfigDir(Paths.get(confDir), confName, excludePattern); } else if (line.getOptionValue(CMD).equalsIgnoreCase(DOWNCONFIG)) { if (!line.hasOption(CONFDIR) || !line.hasOption(CONFNAME)) { System.out.println("-" + CONFDIR + " and -" + CONFNAME diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java index 19cea70d582..81b810bf5bd 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java @@ -100,6 +100,13 @@ public class ZkCLITest extends SolrTestCaseJ4 { log.info("####SETUP_END " + getTestName()); } + @Test + public void testCmdConstants() throws Exception { + assertEquals("upconfig", ZkCLI.UPCONFIG); + assertEquals("excluderegex", ZkCLI.EXCLUDE_REGEX); + assertEquals(ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX, ZkCLI.EXCLUDE_REGEX_DEFAULT); + } + @Test public void testBootstrapWithChroot() throws Exception { String chroot = "/foo/bar"; @@ -193,14 +200,22 @@ public class ZkCLITest extends SolrTestCaseJ4 { // test upconfig String confsetname = "confsetone"; - String[] args = new String[] { - "-zkhost", - zkServer.getZkAddress(), - "-cmd", - "upconfig", - "-confdir", - ExternalPaths.TECHPRODUCTS_CONFIGSET, "-confname", confsetname}; - ZkCLI.main(args); + final String[] upconfigArgs; + if (random().nextBoolean()) { + upconfigArgs = new String[] { + "-zkhost", zkServer.getZkAddress(), + "-cmd", ZkCLI.UPCONFIG, + "-confdir", ExternalPaths.TECHPRODUCTS_CONFIGSET, + "-confname", confsetname}; + } else { + upconfigArgs = new String[] { + "-zkhost", zkServer.getZkAddress(), + "-cmd", ZkCLI.UPCONFIG, + "-"+ZkCLI.EXCLUDE_REGEX, ZkCLI.EXCLUDE_REGEX_DEFAULT, + "-confdir", ExternalPaths.TECHPRODUCTS_CONFIGSET, + "-confname", confsetname}; + } + ZkCLI.main(upconfigArgs); assertTrue(zkClient.exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + confsetname, true)); @@ -208,7 +223,7 @@ public class ZkCLITest extends SolrTestCaseJ4 { // ZkCLI.main(new String[0]); // test linkconfig - args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd", + String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd", "linkconfig", "-collection", "collection1", "-confname", confsetname}; ZkCLI.main(args); @@ -236,8 +251,12 @@ public class ZkCLITest extends SolrTestCaseJ4 { int indexOfRelativePath = sourceFile.getAbsolutePath().lastIndexOf("sample_techproducts_configs" + File.separator + "conf"); String relativePathofFile = sourceFile.getAbsolutePath().substring(indexOfRelativePath + 33, sourceFile.getAbsolutePath().length()); File downloadedFile = new File(confDir,relativePathofFile); - assertTrue(downloadedFile.getAbsolutePath() + " does not exist source:" + sourceFile.getAbsolutePath(), downloadedFile.exists()); - assertTrue("Content didn't change",FileUtils.contentEquals(sourceFile,downloadedFile)); + if (ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_PATTERN.matcher(relativePathofFile).matches()) { + assertFalse(sourceFile.getAbsolutePath() + " exists in ZK, downloaded:" + downloadedFile.getAbsolutePath(), downloadedFile.exists()); + } else { + assertTrue(downloadedFile.getAbsolutePath() + " does not exist source:" + sourceFile.getAbsolutePath(), downloadedFile.exists()); + assertTrue(relativePathofFile+" content changed",FileUtils.contentEquals(sourceFile,downloadedFile)); + } } diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java index 9123c539198..eb87c34782e 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java @@ -30,6 +30,8 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Class that manages named configs in Zookeeper @@ -41,6 +43,9 @@ public class ZkConfigManager { /** ZkNode where named configs are stored */ public static final String CONFIGS_ZKNODE = "/configs"; + public static final String UPLOAD_FILENAME_EXCLUDE_REGEX = "^\\..*$"; + public static final Pattern UPLOAD_FILENAME_EXCLUDE_PATTERN = Pattern.compile(UPLOAD_FILENAME_EXCLUDE_REGEX); + private final SolrZkClient zkClient; /** @@ -51,7 +56,8 @@ public class ZkConfigManager { this.zkClient = zkClient; } - private void uploadToZK(final Path rootPath, final String zkPath) throws IOException { + private void uploadToZK(final Path rootPath, final String zkPath, + final Pattern filenameExclusions) throws IOException { if (!Files.exists(rootPath)) throw new IOException("Path " + rootPath + " does not exist"); @@ -60,8 +66,10 @@ public class ZkConfigManager { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { String filename = file.getFileName().toString(); - if (filename.startsWith(".")) + if (filenameExclusions != null && filenameExclusions.matcher(filename).matches()) { + logger.info("uploadToZK skipping '{}' due to filenameExclusions '{}'", filename, filenameExclusions); return FileVisitResult.CONTINUE; + } String zkNode = createZkNodeName(zkPath, rootPath, file); try { zkClient.makePath(zkNode, file.toFile(), false, true); @@ -118,7 +126,20 @@ public class ZkConfigManager { * if an I/O error occurs or the path does not exist */ public void uploadConfigDir(Path dir, String configName) throws IOException { - uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName); + uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName, UPLOAD_FILENAME_EXCLUDE_PATTERN); + } + + /** + * Upload matching files from a given path to a config in Zookeeper + * @param dir {@link java.nio.file.Path} to the files + * @param configName the name to give the config + * @param filenameExclusions files matching this pattern will not be uploaded + * @throws IOException + * if an I/O error occurs or the path does not exist + */ + public void uploadConfigDir(Path dir, String configName, + Pattern filenameExclusions) throws IOException { + uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName, filenameExclusions); } /** diff --git a/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java b/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java index fd7a0acd0ba..4727bcc8641 100644 --- a/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java +++ b/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java @@ -19,6 +19,7 @@ package org.apache.solr.common.cloud; import com.google.common.base.Charsets; import com.google.common.base.Throwables; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.cloud.ZkCLI; import org.apache.solr.cloud.ZkTestServer; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; @@ -53,6 +54,12 @@ public class TestZkConfigManager extends SolrTestCaseJ4 { zkServer = null; } + @Test + public void testConstants() throws Exception { + assertEquals("/configs", ZkConfigManager.CONFIGS_ZKNODE); + assertEquals("^\\..*$", ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX); + } + @Test public void testUploadConfig() throws IOException {