diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java index cd653e7a6ee..703102f6262 100644 --- a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java @@ -71,15 +71,12 @@ public class DistribPackageStore implements PackageStore { } - - - @Override public Path getRealpath(String path) { if (File.separatorChar == '\\') { path = path.replace('/', File.separatorChar); } - if (path.charAt(0) != File.separatorChar) { + if (!path.isEmpty() && path.charAt(0) != File.separatorChar) { path = File.separator + path; } return new File(this.coreContainer.getResourceLoader().getInstancePath() + @@ -222,7 +219,7 @@ public class DistribPackageStore implements PackageStore { "?meta=true&wt=javabin&omitHeader=true"; boolean nodeHasBlob = false; Object nl = Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(), reqUrl, Utils.JAVABINCONSUMER); - if (Utils.getObjectByPath(nl, false, Arrays.asList("files", getMetaPath())) != null) { + if (Utils.getObjectByPath(nl, false, Arrays.asList("files", path)) != null) { nodeHasBlob = true; } @@ -292,7 +289,7 @@ public class DistribPackageStore implements PackageStore { return; } ew.put("timestamp", getTimeStamp()); - if(metaData != null) + if (metaData != null) metaData.writeMap(ew); } @@ -321,7 +318,7 @@ public class DistribPackageStore implements PackageStore { byte[] bytes = baos.toByteArray(); info.persistToFile(entry.buf, ByteBuffer.wrap(bytes, 0, bytes.length)); tmpFiles.put(entry.getPath(), info); - List nodes = coreContainer.getPackageStoreAPI().shuffledNodes(); + List nodes = coreContainer.getPackageStoreAPI().shuffledNodes(); int i = 0; int FETCHFROM_SRC = 50; String myNodeName = coreContainer.getZkController().getNodeName(); @@ -388,7 +385,14 @@ public class DistribPackageStore implements PackageStore { } if (from == null || "*".equals(from)) { - f.fetchFromAnyNode(); + log.info("Missing file in package store: {}", path); + if (f.fetchFromAnyNode()) { + log.info("Successfully downloaded : {}", path); + return true; + } else{ + log.info("Unable to download file : {}", path); + return false; + } } else { f.fetchFileFromNodeAndPersist(from); diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index bf29b6478a9..8327d3662f9 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -18,6 +18,7 @@ package org.apache.solr.pkg; import java.io.IOException; +import java.io.StringWriter; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collections; @@ -35,7 +36,6 @@ import org.apache.solr.client.solrj.request.beans.Package; import org.apache.solr.common.SolrException; import org.apache.solr.common.annotation.JsonProperty; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.cloud.ZooKeeperException; import org.apache.solr.common.util.CommandOperation; import org.apache.solr.common.util.ReflectMapWriter; @@ -65,7 +65,7 @@ public class PackageAPI { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); final CoreContainer coreContainer; - private final ObjectMapper mapper = new ObjectMapper(); + private final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); private final PackageLoader packageLoader; Packages pkgs; @@ -75,9 +75,14 @@ public class PackageAPI { public PackageAPI(CoreContainer coreContainer, PackageLoader loader) { this.coreContainer = coreContainer; this.packageLoader = loader; - mapper.setAnnotationIntrospector(SolrJacksonAnnotationInspector.INSTANCE); pkgs = new Packages(); SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); + try { + pkgs = readPkgsFromZk(null, null); + } catch (KeeperException |InterruptedException e ) { + pkgs = new Packages(); + //ignore + } try { registerListener(zkClient); } catch (KeeperException | InterruptedException e) { @@ -129,7 +134,7 @@ public class PackageAPI { if (data == null || stat == null) { stat = new Stat(); data = coreContainer.getZkController().getZkClient() - .getData(ZkStateReader.CLUSTER_PROPS, null, stat, true); + .getData(SOLR_PKGS_PATH, null, stat, true); } Packages packages = null; @@ -183,7 +188,6 @@ public class PackageAPI { this.files = addVersion.files; } - @Override public boolean equals(Object obj) { if (obj instanceof PkgVersion) { @@ -194,6 +198,15 @@ public class PackageAPI { } return false; } + + @Override + public String toString() { + try { + return Utils.writeJson(this, new StringWriter(), false).toString() ; + } catch (IOException e) { + throw new RuntimeException(e); + } + } } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java index 88e4e0fb7c8..565ff144d97 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java @@ -53,7 +53,7 @@ public class PackageLoader implements Closeable { private final CoreContainer coreContainer; private final Map packageClassLoaders = new ConcurrentHashMap<>(); - private PackageAPI.Packages myCopy; + private PackageAPI.Packages myCopy = new PackageAPI.Packages(); private PackageAPI packageAPI; @@ -61,7 +61,7 @@ public class PackageLoader implements Closeable { public PackageLoader(CoreContainer coreContainer) { this.coreContainer = coreContainer; packageAPI = new PackageAPI(coreContainer, this); - myCopy = packageAPI.pkgs; + refreshPackageConf(); } @@ -104,6 +104,7 @@ public class PackageLoader implements Closeable { for (SolrCore core : coreContainer.getCores()) { core.getPackageListeners().packagesUpdated(updated); } + myCopy = packageAPI.pkgs; } public Map> getModified(PackageAPI.Packages old, PackageAPI.Packages newPkgs) { @@ -248,6 +249,8 @@ public class PackageLoader implements Closeable { this.version = v; List paths = new ArrayList<>(); for (String file : version.files) { + //ensure that the files are downloaded and available + coreContainer.getPackageStoreAPI().getPackageStore().fetch(file,null); paths.add(coreContainer.getPackageStoreAPI().getPackageStore().getRealpath(file)); } diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 4ff0f7afe20..e5e922431f0 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -57,6 +57,7 @@ import static org.apache.solr.common.params.CommonParams.JAVABIN; import static org.apache.solr.common.params.CommonParams.WT; import static org.apache.solr.core.TestDynamicLoading.getFileContent; import static org.apache.solr.filestore.TestDistribPackageStore.readFile; +import static org.apache.solr.filestore.TestDistribPackageStore.waitForAllNodesHaveFile; @LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG") //@org.apache.lucene.util.LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13822") // leaks files @@ -297,6 +298,17 @@ public class TestPackages extends SolrCloudTestCase { verifyCmponent(cluster.getSolrClient(), COLLECTION_NAME, "requestHandler", "/runtime", "mypkg", "2.1" ); + //we create a new node. This node does not have the packages. But it should download it from another node + JettySolrRunner jetty = cluster.startJettySolrRunner(); + //create a new replica for this collection. it should end up + CollectionAdminRequest.addReplicaToShard(COLLECTION_NAME, "shard1") + .setNrtReplicas(1) + .setNode(jetty.getNodeName()) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(COLLECTION_NAME, 2, 5); + waitForAllNodesHaveFile(cluster,FILE3, + Utils.makeMap(":files:" + FILE3 + ":name", "runtimelibs_v3.jar"), + false); } finally { cluster.shutdown();