From 65c34857f825aabd956a99fb0f622c7648ed4489 Mon Sep 17 00:00:00 2001 From: Erick Erickson Date: Sun, 3 Jul 2016 17:14:25 -0700 Subject: [PATCH] SOLR-9194: Enhance the bin/solr script to perform file operations to/from Zookeeper. More Windows fixes --- .../apache/solr/cloud/SolrCLIZkUtilsTest.java | 33 +++++-------------- .../solr/common/cloud/ZkMaintenanceUtils.java | 11 ++++--- 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java index 91c0126ba10..5f1e1511c90 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java @@ -11,9 +11,9 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; -import java.util.Set; import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.util.SolrCLI; import org.apache.zookeeper.KeeperException; import org.junit.AfterClass; @@ -554,13 +554,6 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase { assertFalse("Should fail when trying to remove /.", res == 0); } - private void getAllKids(String zkRoot, Set paths) throws KeeperException, InterruptedException { - for (String node : zkClient.getChildren(zkRoot, null, true)) { - paths.add(node); - getAllKids(zkRoot + "/" + node, paths); - } - } - // We can use this for testing since the goal is to move "some stuff" up to ZK. // The fact that they're in configsets is irrelevant. private void copyConfigUp(Path configSetDir, String srcConfigSet, String dstConfigName) throws Exception { @@ -597,8 +590,8 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase { void verifyAllFilesAreZNodes(Path fileRoot, String zkRoot) throws IOException { Files.walkFileTree(fileRoot, new SimpleFileVisitor() { void checkPathOnZk(Path path) { - String znode = zkRoot + path.toAbsolutePath().toString().substring(fileRoot.toAbsolutePath().toString().length()); - try { + String znode = ZkMaintenanceUtils.createZkNodeName(zkRoot, fileRoot, path); + try { // It's easier to catch this exception and fail than catch it everywher eles. assertTrue("Should have found " + znode + " on Zookeeper", zkClient.exists(znode, true)); } catch (Exception e) { fail("Caught unexpected exception " + e.getMessage() + " Znode we were checking " + znode); @@ -623,27 +616,17 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase { // Insure that all znodes in first are in second and vice-versa private void verifyZnodesMatch(String first, String second) throws KeeperException, InterruptedException { - verifyFirstNodesInSecond(first, second); - verifyFirstNodesInSecond(second, first); + verifyFirstZNodesInSecond(first, second); + verifyFirstZNodesInSecond(second, first); } - private void verifyFirstNodesInSecond(String first, String second) throws KeeperException, InterruptedException { + // Note, no folderol here with Windows path names. + private void verifyFirstZNodesInSecond(String first, String second) throws KeeperException, InterruptedException { for (String node : zkClient.getChildren(first, null, true)) { String fNode = first + "/" + node; String sNode = second + "/" + node; assertTrue("Node " + sNode + " not found. Exists on " + fNode, zkClient.exists(sNode, true)); - verifyFirstNodesInSecond(fNode, sNode); + verifyFirstZNodesInSecond(fNode, sNode); } } - - public static String createZkNodeName(String zkRoot, Path root, Path file) { - String relativePath = root.relativize(file).toString(); - // Windows shenanigans - String separator = root.getFileSystem().getSeparator(); - if ("\\".equals(separator)) - relativePath = relativePath.replaceAll("\\\\", "/"); - return zkRoot + "/" + relativePath; - } - - } diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java index 2b1ae68e41b..d508aaac28d 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java @@ -38,7 +38,7 @@ import org.slf4j.LoggerFactory; * Class to hold ZK upload/download/move common code. With the advent of the upconfig/downconfig/cp/ls/mv commands * in bin/solr it made sense to keep the individual transfer methods in a central place, so here it is. */ -class ZkMaintenanceUtils { +public class ZkMaintenanceUtils { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private ZkMaintenanceUtils() {} // don't let it be instantiated, all methods are static. @@ -50,7 +50,7 @@ class ZkMaintenanceUtils { * @throws KeeperException Could not perform the Zookeeper operation. * @throws InterruptedException Thread interrupted * @throws SolrServerException zookeeper node has children and recurse not specified. - * @returns an indented list of the znodes suitable for display + * @return an indented list of the znodes suitable for display */ public static String listZnode(SolrZkClient zkClient, String path, Boolean recurse) throws KeeperException, InterruptedException, SolrServerException { String root = path; @@ -335,6 +335,10 @@ class ZkMaintenanceUtils { String separator = root.getFileSystem().getSeparator(); if ("\\".equals(separator)) relativePath = relativePath.replaceAll("\\\\", "/"); + // It's possible that the relative path and file are the same, in which case + // adding the bare slash is A Bad Idea + if (relativePath.length() == 0) return zkRoot; + return zkRoot + "/" + relativePath; } } @@ -358,9 +362,6 @@ class ZkCopier implements ZkMaintenanceUtils.ZkVisitor { public void visit(String path) throws InterruptedException, KeeperException { String finalDestination = dest; if (path.equals(source) == false) finalDestination += "/" + path.substring(source.length() + 1); - if (finalDestination.endsWith("/") || path.endsWith("/")) { - int eoe = 99; - } zkClient.makePath(finalDestination, false, true); zkClient.setData(finalDestination, zkClient.getData(path, null, null, true), true); }