Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-13822: Added a testcase and a bug fix

This commit is contained in:
noble 2019-11-13 17:42:37 +11:00
parent 11fd8d630a
commit 550c7296b6
4 changed files with 47 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
View File

@ -71,15 +71,12 @@ public class DistribPackageStore implements PackageStore {
}
@Override
public Path getRealpath(String path) {
if (File.separatorChar == '\\') {
path = path.replace('/', File.separatorChar);
}
if (path.charAt(0) != File.separatorChar) {
if (!path.isEmpty() && path.charAt(0) != File.separatorChar) {
path = File.separator + path;
}
return new File(this.coreContainer.getResourceLoader().getInstancePath() +
@ -222,7 +219,7 @@ public class DistribPackageStore implements PackageStore {
"?meta=true&wt=javabin&omitHeader=true";
boolean nodeHasBlob = false;
Object nl = Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(), reqUrl, Utils.JAVABINCONSUMER);
if (Utils.getObjectByPath(nl, false, Arrays.asList("files", getMetaPath())) != null) {
if (Utils.getObjectByPath(nl, false, Arrays.asList("files", path)) != null) {
nodeHasBlob = true;
}
@ -292,7 +289,7 @@ public class DistribPackageStore implements PackageStore {
return;
}
ew.put("timestamp", getTimeStamp());
if(metaData != null)
if (metaData != null)
metaData.writeMap(ew);
}
@ -321,7 +318,7 @@ public class DistribPackageStore implements PackageStore {
byte[] bytes = baos.toByteArray();
info.persistToFile(entry.buf, ByteBuffer.wrap(bytes, 0, bytes.length));
tmpFiles.put(entry.getPath(), info);
List<String> nodes = coreContainer.getPackageStoreAPI().shuffledNodes();
List<String> nodes = coreContainer.getPackageStoreAPI().shuffledNodes();
int i = 0;
int FETCHFROM_SRC = 50;
String myNodeName = coreContainer.getZkController().getNodeName();
@ -388,7 +385,14 @@ public class DistribPackageStore implements PackageStore {
}
if (from == null || "*".equals(from)) {
f.fetchFromAnyNode();
log.info("Missing file in package store: {}", path);
if (f.fetchFromAnyNode()) {
log.info("Successfully downloaded : {}", path);
return true;
} else{
log.info("Unable to download file : {}", path);
return false;
}
} else {
f.fetchFileFromNodeAndPersist(from);

23
solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
View File

@ -18,6 +18,7 @@
package org.apache.solr.pkg;
import java.io.IOException;
import java.io.StringWriter;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
@ -35,7 +36,6 @@ import org.apache.solr.client.solrj.request.beans.Package;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.annotation.JsonProperty;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.ZooKeeperException;
import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.common.util.ReflectMapWriter;
@ -65,7 +65,7 @@ public class PackageAPI {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
final CoreContainer coreContainer;
private final ObjectMapper mapper = new ObjectMapper();
private final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper();
private final PackageLoader packageLoader;
Packages pkgs;
@ -75,9 +75,14 @@ public class PackageAPI {
public PackageAPI(CoreContainer coreContainer, PackageLoader loader) {
this.coreContainer = coreContainer;
this.packageLoader = loader;
mapper.setAnnotationIntrospector(SolrJacksonAnnotationInspector.INSTANCE);
pkgs = new Packages();
SolrZkClient zkClient = coreContainer.getZkController().getZkClient();
try {
pkgs = readPkgsFromZk(null, null);
} catch (KeeperException |InterruptedException e ) {
pkgs = new Packages();
//ignore
}
try {
registerListener(zkClient);
} catch (KeeperException | InterruptedException e) {
@ -129,7 +134,7 @@ public class PackageAPI {
if (data == null || stat == null) {
stat = new Stat();
data = coreContainer.getZkController().getZkClient()
.getData(ZkStateReader.CLUSTER_PROPS, null, stat, true);
.getData(SOLR_PKGS_PATH, null, stat, true);
}
Packages packages = null;
@ -183,7 +188,6 @@ public class PackageAPI {
this.files = addVersion.files;
}
@Override
public boolean equals(Object obj) {
if (obj instanceof PkgVersion) {
@ -194,6 +198,15 @@ public class PackageAPI {
}
return false;
}
@Override
public String toString() {
try {
return Utils.writeJson(this, new StringWriter(), false).toString() ;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

7
solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
View File

@ -53,7 +53,7 @@ public class PackageLoader implements Closeable {
private final CoreContainer coreContainer;
private final Map<String, Package> packageClassLoaders = new ConcurrentHashMap<>();
private PackageAPI.Packages myCopy;
private PackageAPI.Packages myCopy = new PackageAPI.Packages();
private PackageAPI packageAPI;
@ -61,7 +61,7 @@ public class PackageLoader implements Closeable {
public PackageLoader(CoreContainer coreContainer) {
this.coreContainer = coreContainer;
packageAPI = new PackageAPI(coreContainer, this);
myCopy = packageAPI.pkgs;
refreshPackageConf();
}
@ -104,6 +104,7 @@ public class PackageLoader implements Closeable {
for (SolrCore core : coreContainer.getCores()) {
core.getPackageListeners().packagesUpdated(updated);
}
myCopy = packageAPI.pkgs;
}
public Map<String, List<PackageAPI.PkgVersion>> getModified(PackageAPI.Packages old, PackageAPI.Packages newPkgs) {
@ -248,6 +249,8 @@ public class PackageLoader implements Closeable {
this.version = v;
List<Path> paths = new ArrayList<>();
for (String file : version.files) {
//ensure that the files are downloaded and available
coreContainer.getPackageStoreAPI().getPackageStore().fetch(file,null);
paths.add(coreContainer.getPackageStoreAPI().getPackageStore().getRealpath(file));
}

12
solr/core/src/test/org/apache/solr/pkg/TestPackages.java
View File

@ -57,6 +57,7 @@ import static org.apache.solr.common.params.CommonParams.JAVABIN;
import static org.apache.solr.common.params.CommonParams.WT;
import static org.apache.solr.core.TestDynamicLoading.getFileContent;
import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
import static org.apache.solr.filestore.TestDistribPackageStore.waitForAllNodesHaveFile;
@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
//@org.apache.lucene.util.LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13822") // leaks files
@ -297,6 +298,17 @@ public class TestPackages extends SolrCloudTestCase {
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "requestHandler", "/runtime",
"mypkg", "2.1" );
//we create a new node. This node does not have the packages. But it should download it from another node
JettySolrRunner jetty = cluster.startJettySolrRunner();
//create a new replica for this collection. it should end up
CollectionAdminRequest.addReplicaToShard(COLLECTION_NAME, "shard1")
.setNrtReplicas(1)
.setNode(jetty.getNodeName())
.process(cluster.getSolrClient());
cluster.waitForActiveCollection(COLLECTION_NAME, 2, 5);
waitForAllNodesHaveFile(cluster,FILE3,
Utils.makeMap(":files:" + FILE3 + ":name", "runtimelibs_v3.jar"),
false);
} finally {
cluster.shutdown();