Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-8938: Add optional -excluderegex argument to ZkCLI.

This commit is contained in:
Christine Poerschke 2016-04-07 10:44:08 +01:00
parent 16c6dd9a2d
commit 835dc33102
5 changed files with 72 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
solr/CHANGES.txt
View File

@ -60,6 +60,7 @@ New Features
graph traversal to find the shortest path(s) in an unweighted, directed graph stored in a
SolrCloud collection. (Joel Bernstein)
* SOLR-8938: Add optional -excluderegex argument to ZkCLI. (Christine Poerschke)
Bug Fixes
----------------------

13
solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
View File

@ -29,7 +29,6 @@ import org.apache.commons.cli.PosixParser;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.OnReconnect;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkConfigManager;
import org.apache.solr.common.cloud.ZkStateReader;
@ -48,6 +47,7 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.util.List;
import java.util.concurrent.TimeoutException;
import java.util.regex.Pattern;
public class ZkCLI {
@ -66,7 +66,9 @@ public class ZkCLI {
private static final String RUNZK = "runzk";
private static final String SOLRHOME = "solrhome";
private static final String BOOTSTRAP = "bootstrap";
private static final String UPCONFIG = "upconfig";
static final String UPCONFIG = "upconfig";
static final String EXCLUDE_REGEX = "excluderegex";
static final String EXCLUDE_REGEX_DEFAULT = ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX;
private static final String COLLECTION = "collection";
private static final String CLEAR = "clear";
private static final String LIST = "list";
@ -120,6 +122,9 @@ public class ZkCLI {
options.addOption("c", COLLECTION, true,
"for " + LINKCONFIG + ": name of the collection");
options.addOption(EXCLUDE_REGEX, true,
"for " + UPCONFIG + ": files matching this regular expression won't be uploaded");
options
.addOption(
"r",
@ -211,13 +216,15 @@ public class ZkCLI {
}
String confDir = line.getOptionValue(CONFDIR);
String confName = line.getOptionValue(CONFNAME);
final String excludeExpr = line.getOptionValue(EXCLUDE_REGEX, EXCLUDE_REGEX_DEFAULT);
if(!ZkController.checkChrootPath(zkServerAddress, true)) {
System.out.println("A chroot was specified in zkHost but the znode doesn't exist. ");
System.exit(1);
}
ZkConfigManager configManager = new ZkConfigManager(zkClient);
configManager.uploadConfigDir(Paths.get(confDir), confName);
final Pattern excludePattern = Pattern.compile(excludeExpr);
configManager.uploadConfigDir(Paths.get(confDir), confName, excludePattern);
} else if (line.getOptionValue(CMD).equalsIgnoreCase(DOWNCONFIG)) {
if (!line.hasOption(CONFDIR) || !line.hasOption(CONFNAME)) {
System.out.println("-" + CONFDIR + " and -" + CONFNAME

41
solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
View File

@ -100,6 +100,13 @@ public class ZkCLITest extends SolrTestCaseJ4 {
log.info("####SETUP_END " + getTestName());
}
@Test
public void testCmdConstants() throws Exception {
assertEquals("upconfig", ZkCLI.UPCONFIG);
assertEquals("excluderegex", ZkCLI.EXCLUDE_REGEX);
assertEquals(ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX, ZkCLI.EXCLUDE_REGEX_DEFAULT);
}
@Test
public void testBootstrapWithChroot() throws Exception {
String chroot = "/foo/bar";
@ -193,14 +200,22 @@ public class ZkCLITest extends SolrTestCaseJ4 {
// test upconfig
String confsetname = "confsetone";
String[] args = new String[] {
"-zkhost",
zkServer.getZkAddress(),
"-cmd",
"upconfig",
"-confdir",
ExternalPaths.TECHPRODUCTS_CONFIGSET, "-confname", confsetname};
ZkCLI.main(args);
final String[] upconfigArgs;
if (random().nextBoolean()) {
upconfigArgs = new String[] {
"-zkhost", zkServer.getZkAddress(),
"-cmd", ZkCLI.UPCONFIG,
"-confdir", ExternalPaths.TECHPRODUCTS_CONFIGSET,
"-confname", confsetname};
} else {
upconfigArgs = new String[] {
"-zkhost", zkServer.getZkAddress(),
"-cmd", ZkCLI.UPCONFIG,
"-"+ZkCLI.EXCLUDE_REGEX, ZkCLI.EXCLUDE_REGEX_DEFAULT,
"-confdir", ExternalPaths.TECHPRODUCTS_CONFIGSET,
"-confname", confsetname};
}
ZkCLI.main(upconfigArgs);
assertTrue(zkClient.exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + confsetname, true));
@ -208,7 +223,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {
// ZkCLI.main(new String[0]);
// test linkconfig
args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
"linkconfig", "-collection", "collection1", "-confname", confsetname};
ZkCLI.main(args);
@ -236,8 +251,12 @@ public class ZkCLITest extends SolrTestCaseJ4 {
int indexOfRelativePath = sourceFile.getAbsolutePath().lastIndexOf("sample_techproducts_configs" + File.separator + "conf");
String relativePathofFile = sourceFile.getAbsolutePath().substring(indexOfRelativePath + 33, sourceFile.getAbsolutePath().length());
File downloadedFile = new File(confDir,relativePathofFile);
assertTrue(downloadedFile.getAbsolutePath() + " does not exist source:" + sourceFile.getAbsolutePath(), downloadedFile.exists());
assertTrue("Content didn't change",FileUtils.contentEquals(sourceFile,downloadedFile));
if (ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_PATTERN.matcher(relativePathofFile).matches()) {
assertFalse(sourceFile.getAbsolutePath() + " exists in ZK, downloaded:" + downloadedFile.getAbsolutePath(), downloadedFile.exists());
} else {
assertTrue(downloadedFile.getAbsolutePath() + " does not exist source:" + sourceFile.getAbsolutePath(), downloadedFile.exists());
assertTrue(relativePathofFile+" content changed",FileUtils.contentEquals(sourceFile,downloadedFile));
}
}

27
solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java
View File

@ -30,6 +30,8 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Class that manages named configs in Zookeeper
@ -41,6 +43,9 @@ public class ZkConfigManager {
/** ZkNode where named configs are stored */
public static final String CONFIGS_ZKNODE = "/configs";
public static final String UPLOAD_FILENAME_EXCLUDE_REGEX = "^\\..*$";
public static final Pattern UPLOAD_FILENAME_EXCLUDE_PATTERN = Pattern.compile(UPLOAD_FILENAME_EXCLUDE_REGEX);
private final SolrZkClient zkClient;
/**
@ -51,7 +56,8 @@ public class ZkConfigManager {
this.zkClient = zkClient;
}
private void uploadToZK(final Path rootPath, final String zkPath) throws IOException {
private void uploadToZK(final Path rootPath, final String zkPath,
final Pattern filenameExclusions) throws IOException {
if (!Files.exists(rootPath))
throw new IOException("Path " + rootPath + " does not exist");
@ -60,8 +66,10 @@ public class ZkConfigManager {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
String filename = file.getFileName().toString();
if (filename.startsWith("."))
if (filenameExclusions != null && filenameExclusions.matcher(filename).matches()) {
logger.info("uploadToZK skipping '{}' due to filenameExclusions '{}'", filename, filenameExclusions);
return FileVisitResult.CONTINUE;
}
String zkNode = createZkNodeName(zkPath, rootPath, file);
try {
zkClient.makePath(zkNode, file.toFile(), false, true);
@ -118,7 +126,20 @@ public class ZkConfigManager {
* if an I/O error occurs or the path does not exist
*/
public void uploadConfigDir(Path dir, String configName) throws IOException {
uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName);
uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName, UPLOAD_FILENAME_EXCLUDE_PATTERN);
}
/**
* Upload matching files from a given path to a config in Zookeeper
* @param dir {@link java.nio.file.Path} to the files
* @param configName the name to give the config
* @param filenameExclusions files matching this pattern will not be uploaded
* @throws IOException
* if an I/O error occurs or the path does not exist
*/
public void uploadConfigDir(Path dir, String configName,
Pattern filenameExclusions) throws IOException {
uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName, filenameExclusions);
}
/**

7
solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java
View File

@ -19,6 +19,7 @@ package org.apache.solr.common.cloud;
import com.google.common.base.Charsets;
import com.google.common.base.Throwables;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.cloud.ZkCLI;
import org.apache.solr.cloud.ZkTestServer;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
@ -53,6 +54,12 @@ public class TestZkConfigManager extends SolrTestCaseJ4 {
zkServer = null;
}
@Test
public void testConstants() throws Exception {
assertEquals("/configs", ZkConfigManager.CONFIGS_ZKNODE);
assertEquals("^\\..*$", ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX);
}
@Test
public void testUploadConfig() throws IOException {