SOLR-13822: Added a testcase and a bug fix

solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java

@@ -71,15 +71,12 @@ public class DistribPackageStore implements PackageStore {
   }
 
 
-
-
-
   @Override
   public Path getRealpath(String path) {
     if (File.separatorChar == '\\') {
       path = path.replace('/', File.separatorChar);
     }
-    if (path.charAt(0) != File.separatorChar) {
+    if (!path.isEmpty() && path.charAt(0) != File.separatorChar) {
       path = File.separator + path;
     }
     return new File(this.coreContainer.getResourceLoader().getInstancePath() +
@@ -222,7 +219,7 @@ public class DistribPackageStore implements PackageStore {
               "?meta=true&wt=javabin&omitHeader=true";
           boolean nodeHasBlob = false;
           Object nl = Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(), reqUrl, Utils.JAVABINCONSUMER);
-          if (Utils.getObjectByPath(nl, false, Arrays.asList("files", getMetaPath())) != null) {
+          if (Utils.getObjectByPath(nl, false, Arrays.asList("files", path)) != null) {
             nodeHasBlob = true;
           }
 
@@ -292,7 +289,7 @@ public class DistribPackageStore implements PackageStore {
             return;
           }
           ew.put("timestamp", getTimeStamp());
-          if(metaData != null)
+          if (metaData != null)
             metaData.writeMap(ew);
 
         }
@@ -388,7 +385,14 @@ public class DistribPackageStore implements PackageStore {
     }
 
     if (from == null || "*".equals(from)) {
-      f.fetchFromAnyNode();
+      log.info("Missing file in package store: {}", path);
+      if (f.fetchFromAnyNode()) {
+        log.info("Successfully downloaded : {}", path);
+        return true;
+      } else{
+        log.info("Unable to download file : {}", path);
+        return false;
+      }
 
     } else {
       f.fetchFileFromNodeAndPersist(from);

solr/core/src/java/org/apache/solr/pkg/PackageAPI.java

@@ -18,6 +18,7 @@
 package org.apache.solr.pkg;
 
 import java.io.IOException;
+import java.io.StringWriter;
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -35,7 +36,6 @@ import org.apache.solr.client.solrj.request.beans.Package;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.annotation.JsonProperty;
 import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.cloud.ZooKeeperException;
 import org.apache.solr.common.util.CommandOperation;
 import org.apache.solr.common.util.ReflectMapWriter;
@@ -65,7 +65,7 @@ public class PackageAPI {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
   final CoreContainer coreContainer;
-  private final ObjectMapper mapper = new ObjectMapper();
+  private final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper();
   private final PackageLoader packageLoader;
   Packages pkgs;
 
@@ -75,9 +75,14 @@ public class PackageAPI {
   public PackageAPI(CoreContainer coreContainer, PackageLoader loader) {
     this.coreContainer = coreContainer;
     this.packageLoader = loader;
-    mapper.setAnnotationIntrospector(SolrJacksonAnnotationInspector.INSTANCE);
     pkgs = new Packages();
     SolrZkClient zkClient = coreContainer.getZkController().getZkClient();
+    try {
+      pkgs = readPkgsFromZk(null, null);
+    } catch (KeeperException |InterruptedException e ) {
+      pkgs = new Packages();
+      //ignore
+    }
     try {
       registerListener(zkClient);
     } catch (KeeperException | InterruptedException e) {
@@ -129,7 +134,7 @@ public class PackageAPI {
     if (data == null || stat == null) {
       stat = new Stat();
       data = coreContainer.getZkController().getZkClient()
-          .getData(ZkStateReader.CLUSTER_PROPS, null, stat, true);
+          .getData(SOLR_PKGS_PATH, null, stat, true);
 
     }
     Packages packages = null;
@@ -183,7 +188,6 @@ public class PackageAPI {
       this.files = addVersion.files;
     }
 
-
     @Override
     public boolean equals(Object obj) {
       if (obj instanceof PkgVersion) {
@@ -194,6 +198,15 @@ public class PackageAPI {
       }
       return false;
     }
+
+    @Override
+    public String toString() {
+      try {
+        return Utils.writeJson(this, new StringWriter(), false).toString() ;
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
   }
 
 

solr/core/src/java/org/apache/solr/pkg/PackageLoader.java

@@ -53,7 +53,7 @@ public class PackageLoader implements Closeable {
   private final CoreContainer coreContainer;
   private final Map<String, Package> packageClassLoaders = new ConcurrentHashMap<>();
 
-  private PackageAPI.Packages myCopy;
+  private PackageAPI.Packages myCopy =  new PackageAPI.Packages();
 
   private PackageAPI packageAPI;
 
@@ -61,7 +61,7 @@ public class PackageLoader implements Closeable {
   public PackageLoader(CoreContainer coreContainer) {
     this.coreContainer = coreContainer;
     packageAPI = new PackageAPI(coreContainer, this);
-    myCopy = packageAPI.pkgs;
+    refreshPackageConf();
 
   }
 
@@ -104,6 +104,7 @@ public class PackageLoader implements Closeable {
     for (SolrCore core : coreContainer.getCores()) {
       core.getPackageListeners().packagesUpdated(updated);
     }
+    myCopy = packageAPI.pkgs;
   }
 
   public Map<String, List<PackageAPI.PkgVersion>> getModified(PackageAPI.Packages old, PackageAPI.Packages newPkgs) {
@@ -248,6 +249,8 @@ public class PackageLoader implements Closeable {
         this.version = v;
         List<Path> paths = new ArrayList<>();
         for (String file : version.files) {
+          //ensure that the files are downloaded and available
+          coreContainer.getPackageStoreAPI().getPackageStore().fetch(file,null);
           paths.add(coreContainer.getPackageStoreAPI().getPackageStore().getRealpath(file));
         }
 

solr/core/src/test/org/apache/solr/pkg/TestPackages.java

@@ -57,6 +57,7 @@ import static org.apache.solr.common.params.CommonParams.JAVABIN;
 import static org.apache.solr.common.params.CommonParams.WT;
 import static org.apache.solr.core.TestDynamicLoading.getFileContent;
 import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
+import static org.apache.solr.filestore.TestDistribPackageStore.waitForAllNodesHaveFile;
 
 @LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
 //@org.apache.lucene.util.LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13822") // leaks files
@@ -297,6 +298,17 @@ public class TestPackages extends SolrCloudTestCase {
       verifyCmponent(cluster.getSolrClient(),
           COLLECTION_NAME, "requestHandler", "/runtime",
           "mypkg", "2.1" );
+      //we create a new node. This node does not have the packages. But it should download it from another node
+      JettySolrRunner jetty = cluster.startJettySolrRunner();
+      //create a new replica for this collection. it should end up
+      CollectionAdminRequest.addReplicaToShard(COLLECTION_NAME, "shard1")
+          .setNrtReplicas(1)
+          .setNode(jetty.getNodeName())
+          .process(cluster.getSolrClient());
+      cluster.waitForActiveCollection(COLLECTION_NAME, 2, 5);
+      waitForAllNodesHaveFile(cluster,FILE3,
+          Utils.makeMap(":files:" + FILE3 + ":name", "runtimelibs_v3.jar"),
+          false);
 
     } finally {
       cluster.shutdown();