Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-02-08 19:15:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

SOLR-9194: Enhance the bin/solr script to perform file operations to/from Zookeeper - Fix Windows failures

Browse Source
This commit is contained in:
Erick Erickson 2016-07-02 16:07:38 -07:00
parent ee0dbf55f7
commit dd761584c7
3 changed files with 68 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
solr/core/src/java/org/apache/solr/util/SolrCLI.java
View File

@ -1441,7 +1441,7 @@ public class SolrCLI {
} else if (configExistsInZk) { } else if (configExistsInZk) {
echo("Re-using existing configuration directory "+confname); echo("Re-using existing configuration directory "+confname);
} else { } else {
Path confPath = ZkConfigManager.getConfigsetPath(confname, cli.getOptionValue("confdir", DEFAULT_CONFIG_SET), Path confPath = ZkConfigManager.getConfigsetPath(cli.getOptionValue("confdir", DEFAULT_CONFIG_SET),
cli.getOptionValue("configsetsDir")); cli.getOptionValue("configsetsDir"));
echo("Uploading " + confPath.toAbsolutePath().toString() + echo("Uploading " + confPath.toAbsolutePath().toString() +
@ -1687,7 +1687,7 @@ public class SolrCLI {
OptionBuilder OptionBuilder
.withArgName("configsetsDir") .withArgName("configsetsDir")
.hasArg() .hasArg()
.isRequired() .isRequired(false)
.withDescription("Parent directory of example configsets") .withDescription("Parent directory of example configsets")
.create("configsetsDir"), .create("configsetsDir"),
OptionBuilder OptionBuilder
@ -1714,7 +1714,7 @@ public class SolrCLI {
String confName = cli.getOptionValue("confname"); String confName = cli.getOptionValue("confname");
try (SolrZkClient zkClient = new SolrZkClient(zkHost, 30000)) { try (SolrZkClient zkClient = new SolrZkClient(zkHost, 30000)) {
echo("\nConnecting to ZooKeeper at " + zkHost + " ..."); echo("\nConnecting to ZooKeeper at " + zkHost + " ...");
Path confPath = ZkConfigManager.getConfigsetPath(confName, cli.getOptionValue("confdir"), cli.getOptionValue("configsetsDir")); Path confPath = ZkConfigManager.getConfigsetPath(cli.getOptionValue("confdir"), cli.getOptionValue("configsetsDir"));
echo("Uploading " + confPath.toAbsolutePath().toString() + echo("Uploading " + confPath.toAbsolutePath().toString() +
" for config " + cli.getOptionValue("confname") + " to ZooKeeper at " + zkHost); " for config " + cli.getOptionValue("confname") + " to ZooKeeper at " + zkHost);

80
solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java
View File

@ -60,11 +60,12 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
@Test @Test
public void testUpconfig() throws Exception { public void testUpconfig() throws Exception {
// Use a full, explicit path for configset. // Use a full, explicit path for configset.
Path src = TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf");
Path configSet = TEST_PATH().resolve("configsets").resolve("cloud-subdirs"); Path configSet = TEST_PATH().resolve("configsets");
copyConfigUp(src, configSet, "upconfig1"); Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
copyConfigUp(configSet, "cloud-subdirs", "upconfig1");
// Now do we have that config up on ZK? // Now do we have that config up on ZK?
verifyZkLocalPathsMatch(src, "/configs/upconfig1"); verifyZkLocalPathsMatch(srcPathCheck, "/configs/upconfig1");
// Now just use a name in the configsets directory, do we find it? // Now just use a name in the configsets directory, do we find it?
configSet = TEST_PATH().resolve("configsets"); configSet = TEST_PATH().resolve("configsets");
@ -81,7 +82,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
int res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); int res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
assertEquals("tool should have returned 0 for success ", 0, res); assertEquals("tool should have returned 0 for success ", 0, res);
// Now do we have that config up on ZK? // Now do we have that config up on ZK?
verifyZkLocalPathsMatch(src, "/configs/upconfig2"); verifyZkLocalPathsMatch(srcPathCheck, "/configs/upconfig2");
// do we barf on a bogus path? // do we barf on a bogus path?
args = new String[]{ args = new String[]{
@ -104,11 +105,12 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
Path tmp = createTempDir("downConfigNewPlace"); Path tmp = createTempDir("downConfigNewPlace");
// First we need a configset on ZK to bring down. // First we need a configset on ZK to bring down.
Path src = TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf");
Path configSet = TEST_PATH().resolve("configsets").resolve("cloud-subdirs"); Path configSet = TEST_PATH().resolve("configsets");
copyConfigUp(src, configSet, "downconfig1"); Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
copyConfigUp(configSet, "cloud-subdirs", "downconfig1");
// Now do we have that config up on ZK? // Now do we have that config up on ZK?
verifyZkLocalPathsMatch(src, "/configs/downconfig1"); verifyZkLocalPathsMatch(srcPathCheck, "/configs/downconfig1");
String[] args = new String[]{ String[] args = new String[]{
"-confname", "downconfig1", "-confname", "downconfig1",
@ -126,10 +128,11 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
@Test @Test
public void testCp() throws Exception { public void testCp() throws Exception {
// First get something up on ZK // First get something up on ZK
Path src = TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf");
Path configSet = TEST_PATH().resolve("configsets").resolve("cloud-subdirs");
copyConfigUp(src, configSet, "cp1"); Path configSet = TEST_PATH().resolve("configsets");
Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
copyConfigUp(configSet, "cloud-subdirs", "cp1");
// Now copy it somewhere else on ZK. // Now copy it somewhere else on ZK.
String[] args = new String[]{ String[] args = new String[]{
@ -176,7 +179,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
// try with local->zk // try with local->zk
args = new String[]{ args = new String[]{
"-src", src.toAbsolutePath().toString(), "-src", srcPathCheck.toAbsolutePath().toString(),
"-dst", "zk:/cp3", "-dst", "zk:/cp3",
"-recurse", "true", "-recurse", "true",
"-zkHost", zkAddr, "-zkHost", zkAddr,
@ -184,11 +187,11 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
assertEquals("Copy should have succeeded.", 0, res); assertEquals("Copy should have succeeded.", 0, res);
verifyZkLocalPathsMatch(src, "/cp3"); verifyZkLocalPathsMatch(srcPathCheck, "/cp3");
// try with local->zk, file: specified // try with local->zk, file: specified
args = new String[]{ args = new String[]{
"-src", "file:" + src.toAbsolutePath().toString(), "-src", "file:" + srcPathCheck.toAbsolutePath().toString(),
"-dst", "zk:/cp4", "-dst", "zk:/cp4",
"-recurse", "true", "-recurse", "true",
"-zkHost", zkAddr, "-zkHost", zkAddr,
@ -196,12 +199,12 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
assertEquals("Copy should have succeeded.", 0, res); assertEquals("Copy should have succeeded.", 0, res);
verifyZkLocalPathsMatch(src, "/cp4"); verifyZkLocalPathsMatch(srcPathCheck, "/cp4");
// try with recurse not specified // try with recurse not specified
args = new String[]{ args = new String[]{
"-src", "file:" + src.toAbsolutePath().toString(), "-src", "file:" + srcPathCheck.toAbsolutePath().toString(),
"-dst", "zk:/cp5Fail", "-dst", "zk:/cp5Fail",
"-zkHost", zkAddr, "-zkHost", zkAddr,
}; };
@ -211,7 +214,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
// try with recurse = false // try with recurse = false
args = new String[]{ args = new String[]{
"-src", "file:" + src.toAbsolutePath().toString(), "-src", "file:" + srcPathCheck.toAbsolutePath().toString(),
"-dst", "zk:/cp6Fail", "-dst", "zk:/cp6Fail",
"-recurse", "false", "-recurse", "false",
"-zkHost", zkAddr, "-zkHost", zkAddr,
@ -241,7 +244,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
// copy to ZK ending in '/'. // copy to ZK ending in '/'.
//src and cp3 are valid //src and cp3 are valid
args = new String[]{ args = new String[]{
"-src", "file:" + src.normalize().toAbsolutePath().toString() + "/solrconfig.xml", "-src", "file:" + srcPathCheck.normalize().toAbsolutePath().toString() + "/solrconfig.xml",
"-dst", "zk:/powerup/", "-dst", "zk:/powerup/",
"-recurse", "false", "-recurse", "false",
"-zkHost", zkAddr, "-zkHost", zkAddr,
@ -254,7 +257,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
// copy individual file up // copy individual file up
//src and cp3 are valid //src and cp3 are valid
args = new String[]{ args = new String[]{
"-src", "file:" + src.normalize().toAbsolutePath().toString() + "/solrconfig.xml", "-src", "file:" + srcPathCheck.normalize().toAbsolutePath().toString() + "/solrconfig.xml",
"-dst", "zk:/copyUpFile.xml", "-dst", "zk:/copyUpFile.xml",
"-recurse", "false", "-recurse", "false",
"-zkHost", zkAddr, "-zkHost", zkAddr,
@ -309,10 +312,11 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
public void testMv() throws Exception { public void testMv() throws Exception {
// First get something up on ZK // First get something up on ZK
Path src = TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf");
Path configSet = TEST_PATH().resolve("configsets").resolve("cloud-subdirs");
copyConfigUp(src, configSet, "mv1"); Path configSet = TEST_PATH().resolve("configsets");
Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
copyConfigUp(configSet, "cloud-subdirs", "mv1");
// Now move it somewhere else. // Now move it somewhere else.
String[] args = new String[]{ String[] args = new String[]{
@ -327,7 +331,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
assertEquals("Move should have succeeded.", 0, res); assertEquals("Move should have succeeded.", 0, res);
// Now does the moved directory match the original on disk? // Now does the moved directory match the original on disk?
verifyZkLocalPathsMatch(src, "/mv2"); verifyZkLocalPathsMatch(srcPathCheck, "/mv2");
// And are we sure the old path is gone? // And are we sure the old path is gone?
assertFalse("/configs/mv1 Znode should not be there: ", zkClient.exists("/configs/mv1", true)); assertFalse("/configs/mv1 Znode should not be there: ", zkClient.exists("/configs/mv1", true));
@ -356,7 +360,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
assertFalse("Znode /mv3 really should be gone", zkClient.exists("/mv3", true)); assertFalse("Znode /mv3 really should be gone", zkClient.exists("/mv3", true));
// Now does the moved directory match the original on disk? // Now does the moved directory match the original on disk?
verifyZkLocalPathsMatch(src, "/mv4"); verifyZkLocalPathsMatch(srcPathCheck, "/mv4");
args = new String[]{ args = new String[]{
"-src", "/mv4/solrconfig.xml", "-src", "/mv4/solrconfig.xml",
@ -387,10 +391,9 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
@Test @Test
public void testLs() throws Exception { public void testLs() throws Exception {
Path src = TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf"); Path configSet = TEST_PATH().resolve("configsets");
Path configSet = TEST_PATH().resolve("configsets").resolve("cloud-subdirs");
copyConfigUp(src, configSet, "lister"); copyConfigUp(configSet, "cloud-subdirs", "lister");
// Should only find a single level. // Should only find a single level.
String[] args = new String[]{ String[] args = new String[]{
@ -485,11 +488,12 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
@Test @Test
public void testRm() throws Exception { public void testRm() throws Exception {
Path src = TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf");
Path configSet = TEST_PATH().resolve("configsets").resolve("cloud-subdirs");
copyConfigUp(src, configSet, "rm1"); Path configSet = TEST_PATH().resolve("configsets");
copyConfigUp(src, configSet, "rm2"); Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
copyConfigUp(configSet, "cloud-subdirs", "rm1");
copyConfigUp(configSet, "cloud-subdirs", "rm2");
// Should fail if recurse not set. // Should fail if recurse not set.
String[] args = new String[]{ String[] args = new String[]{
@ -504,7 +508,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
assertTrue("Should have failed to remove node with children unless -recurse is set to true", res != 0); assertTrue("Should have failed to remove node with children unless -recurse is set to true", res != 0);
// Are we sure all the znodes are still there? // Are we sure all the znodes are still there?
verifyZkLocalPathsMatch(src, "/configs/rm1"); verifyZkLocalPathsMatch(srcPathCheck, "/configs/rm1");
args = new String[]{ args = new String[]{
"-path", "zk:/configs/rm1", "-path", "zk:/configs/rm1",
@ -545,7 +549,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
"-zkHost", zkAddr, "-zkHost", zkAddr,
}; };
copyConfigUp(src, configSet, "rm3"); copyConfigUp(configSet, "cloud-subdirs", "rm3");
res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
assertFalse("Should fail when trying to remove /.", res == 0); assertFalse("Should fail when trying to remove /.", res == 0);
} }
@ -559,12 +563,12 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
// We can use this for testing since the goal is to move "some stuff" up to ZK. // We can use this for testing since the goal is to move "some stuff" up to ZK.
// The fact that they're in configsets is irrelevant. // The fact that they're in configsets is irrelevant.
private void copyConfigUp(Path src, Path configSet, String confName) throws Exception { private void copyConfigUp(Path configSetDir, String srcConfigSet, String dstConfigName) throws Exception {
String[] args = new String[]{ String[] args = new String[]{
"-confname", confName, "-confname", dstConfigName,
"-confdir", src.toAbsolutePath().toString(), "-confdir", srcConfigSet,
"-zkHost", zkAddr, "-zkHost", zkAddr,
"-configsetsDir", configSet.toAbsolutePath().toString(), "-configsetsDir", configSetDir.toAbsolutePath().toString(),
}; };
SolrCLI.ConfigSetUploadTool tool = new SolrCLI.ConfigSetUploadTool(); SolrCLI.ConfigSetUploadTool tool = new SolrCLI.ConfigSetUploadTool();

37
solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java
View File

@ -180,34 +180,41 @@ public class ZkConfigManager {
// This method is used by configSetUploadTool and CreateTool to resolve the configset directory. // This method is used by configSetUploadTool and CreateTool to resolve the configset directory.
// Check several possibilities: // Check several possibilities:
// 1> configSetDirParam + confname is defined in configsets, thus has a conf/solrconfig.xml // 1> confDir/solrconfig.xml exists
// 2> configSet is a path that contains conf/solrconfig.xml // 2> confDir/conf/solrconfig.xml exists
// 3> configSet is t // 3> configSetDir/confDir/conf/solrconfig.xml exists (canned configs)
public static Path getConfigsetPath(String confname, String configSet, String configSetDirParam) throws IOException {
// one of the canned configsets. // Order is important here since "confDir" may be
Path ret = Paths.get(configSetDirParam, configSet, "conf", "solrconfig.xml").normalize(); // 1> a full path to the parent of a solrconfig.xml or parent of /conf/solrconfig.xml
// 2> one of the canned config sets only, e.g. basic_configs
// and trying to assemble a path for configsetDir/confDir is A Bad Idea. if confDir is a full path.
public static Path getConfigsetPath(String confDir, String configSetDir) throws IOException {
// A local path to the source, probably already includes "conf".
Path ret = Paths.get(confDir, "solrconfig.xml").normalize();
if (Files.exists(ret)) { if (Files.exists(ret)) {
return Paths.get(configSetDirParam, configSet, "conf").normalize(); return Paths.get(confDir).normalize();
} }
// a local path to the parent of a "conf" directory // a local path to the parent of a "conf" directory
ret = Paths.get(configSet, "conf", "solrconfig.xml").normalize(); ret = Paths.get(confDir, "conf", "solrconfig.xml").normalize();
if (Files.exists(ret)) { if (Files.exists(ret)) {
return Paths.get(configSet, "conf").normalize(); return Paths.get(confDir, "conf").normalize();
} }
// A local path to the source, probably already includes "conf". // one of the canned configsets.
ret = Paths.get(configSet, "solrconfig.xml").normalize(); ret = Paths.get(configSetDir, confDir, "conf", "solrconfig.xml").normalize();
if (Files.exists(ret)) { if (Files.exists(ret)) {
return Paths.get(configSet).normalize(); return Paths.get(configSetDir, confDir, "conf").normalize();
} }
throw new IllegalArgumentException(String.format(Locale.ROOT, throw new IllegalArgumentException(String.format(Locale.ROOT,
"Could not find solrconfig.xml at %s, %s or %s", "Could not find solrconfig.xml at %s, %s or %s",
Paths.get(configSetDirParam, configSet, "conf", "solrconfig.xml").normalize().toAbsolutePath().toString(), Paths.get(configSetDir, "solrconfig.xml").normalize().toAbsolutePath().toString(),
Paths.get(configSet, "conf", "solrconfig.xml").normalize().toAbsolutePath().toString(), Paths.get(configSetDir, "conf", "solrconfig.xml").normalize().toAbsolutePath().toString(),
Paths.get(configSet, "solrconfig.xml").normalize().toAbsolutePath().toString() Paths.get(configSetDir, confDir, "conf", "solrconfig.xml").normalize().toAbsolutePath().toString()
)); ));
} }
} }