From 4d9d0c0011de8653fbe404af702e822883e8fb13 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Tue, 23 Feb 2016 16:14:49 +0530 Subject: [PATCH] SOLR-8719 renamed classes to make their usage generic --- ...JarRepository.java => BlobRepository.java} | 114 +++++++++++------- .../org/apache/solr/core/CoreContainer.java | 6 +- .../java/org/apache/solr/core/PluginBag.java | 10 +- 3 files changed, 81 insertions(+), 49 deletions(-) rename solr/core/src/java/org/apache/solr/core/{JarRepository.java => BlobRepository.java} (69%) diff --git a/solr/core/src/java/org/apache/solr/core/JarRepository.java b/solr/core/src/java/org/apache/solr/core/BlobRepository.java similarity index 69% rename from solr/core/src/java/org/apache/solr/core/JarRepository.java rename to solr/core/src/java/org/apache/solr/core/BlobRepository.java index c760b594927..67398269113 100644 --- a/solr/core/src/java/org/apache/solr/core/JarRepository.java +++ b/solr/core/src/java/org/apache/solr/core/BlobRepository.java @@ -21,6 +21,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.lang.invoke.MethodHandles; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -46,13 +47,14 @@ import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.util.CryptoKeys; import org.apache.solr.util.SimplePostTool; +import org.apache.zookeeper.server.ByteBufferInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * The purpose of this class is to store the Jars loaded in memory and to keep only one copy of the Jar in a single node. */ -public class JarRepository { +public class BlobRepository { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); static final Random RANDOM; @@ -68,21 +70,44 @@ public class JarRepository { } private final CoreContainer coreContainer; - private Map jars = new ConcurrentHashMap<>(); + private Map blobs = new ConcurrentHashMap<>(); - public JarRepository(CoreContainer coreContainer) { + public BlobRepository(CoreContainer coreContainer) { this.coreContainer = coreContainer; } + public static ByteBuffer getFileContent(BlobContent blobContent, String entryName) throws IOException { + ByteArrayInputStream zipContents = new ByteArrayInputStream(blobContent.buffer.array(), blobContent.buffer.arrayOffset(), blobContent.buffer.limit()); + ZipInputStream zis = new ZipInputStream(zipContents); + try { + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + if (entryName == null || entryName.equals(entry.getName())) { + SimplePostTool.BAOS out = new SimplePostTool.BAOS(); + byte[] buffer = new byte[2048]; + int size; + while ((size = zis.read(buffer, 0, buffer.length)) != -1) { + out.write(buffer, 0, size); + } + out.close(); + return out.getByteBuffer(); + } + } + } finally { + zis.closeEntry(); + } + return null; + } + /** * Returns the contents of a jar and increments a reference count. Please return the same object to decrease the refcount * * @param key it is a combination of blobname and version like blobName/version * @return The reference of a jar */ - public JarContentRef getJarIncRef(String key) { - JarContent jar = jars.get(key); - if (jar == null) { + public BlobContentRef getBlobIncRef(String key) { + BlobContent aBlob = blobs.get(key); + if (aBlob == null) { if (this.coreContainer.isZooKeeperAware()) { Replica replica = getSystemCollReplica(); String url = replica.getStr(BASE_URL_PROP) + "/.system/blob/" + key + "?wt=filestream"; @@ -106,7 +131,7 @@ public class JarRepository { } finally { httpGet.releaseConnection(); } - jars.put(key, jar = new JarContent(key, b)); + blobs.put(key, aBlob = new BlobContent(key, b)); } else { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Jar loading is not supported in non-cloud mode"); // todo @@ -114,9 +139,9 @@ public class JarRepository { } - JarContentRef ref = new JarContentRef(jar); - synchronized (jar.references) { - jar.references.add(ref); + BlobContentRef ref = new BlobContentRef(aBlob); + synchronized (aBlob.references) { + aBlob.references.add(ref); } return ref; @@ -157,52 +182,54 @@ public class JarRepository { * * @param ref The reference that is already there. Doing multiple calls with same ref will not matter */ - public void decrementJarRefCount(JarContentRef ref) { + public void decrementBlobRefCount(BlobContentRef ref) { if (ref == null) return; - synchronized (ref.jar.references) { - if (!ref.jar.references.remove(ref)) { + synchronized (ref.blob.references) { + if (!ref.blob.references.remove(ref)) { log.error("Multiple releases for the same reference"); } - if (ref.jar.references.isEmpty()) { - jars.remove(ref.jar.key); + if (ref.blob.references.isEmpty()) { + blobs.remove(ref.blob.key); } } } - public static class JarContent { + public static class BlobContent { private final String key; + private Map decodedObjects = null; // TODO move this off-heap private final ByteBuffer buffer; // ref counting mechanism - private final Set references = new HashSet<>(); + private final Set references = new HashSet<>(); - public JarContent(String key, ByteBuffer buffer) { + + public BlobContent(String key, ByteBuffer buffer) { this.key = key; this.buffer = buffer; } - public ByteBuffer getFileContent(String entryName) throws IOException { - ByteArrayInputStream zipContents = new ByteArrayInputStream(buffer.array(), buffer.arrayOffset(), buffer.limit()); - ZipInputStream zis = new ZipInputStream(zipContents); - try { - ZipEntry entry; - while ((entry = zis.getNextEntry()) != null) { - if (entryName == null || entryName.equals(entry.getName())) { - SimplePostTool.BAOS out = new SimplePostTool.BAOS(); - byte[] buffer = new byte[2048]; - int size; - while ((size = zis.read(buffer, 0, buffer.length)) != -1) { - out.write(buffer, 0, size); - } - out.close(); - return out.getByteBuffer(); - } + /** + * This method decodes the byte[] to a custom Object + * + * @param key The key is used to store the decoded Object. it is possible to have multiple + * decoders for the same blob (may be unusual). + * @param decoder A decoder instance + * @return the decoded Object . If it was already decoded, then return from the cache + */ + public T decodeAndCache(String key, Decoder decoder) { + if (decodedObjects == null) { + synchronized (this) { + if (decodedObjects == null) decodedObjects = new ConcurrentHashMap<>(); } - } finally { - zis.closeEntry(); } - return null; + + Object t = decodedObjects.get(key); + if (t != null) return (T) t; + t = decoder.decode(new ByteBufferInputStream(buffer)); + decodedObjects.put(key, t); + return (T) t; + } public String checkSignature(String base64Sig, CryptoKeys keys) { @@ -211,11 +238,16 @@ public class JarRepository { } - public static class JarContentRef { - public final JarContent jar; + public interface Decoder { - private JarContentRef(JarContent jar) { - this.jar = jar; + T decode(InputStream inputStream); + } + + public static class BlobContentRef { + public final BlobContent blob; + + private BlobContentRef(BlobContent blob) { + this.blob = blob; } } diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 770372173ac..7a55e053d78 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -134,7 +134,7 @@ public class CoreContainer { private String hostName; - private final JarRepository jarRepository = new JarRepository(this); + private final BlobRepository blobRepository = new BlobRepository(this); private PluginBag containerHandlers = new PluginBag<>(SolrRequestHandler.class, null); @@ -1098,8 +1098,8 @@ public class CoreContainer { return core; } - public JarRepository getJarRepository(){ - return jarRepository; + public BlobRepository getBlobRepository(){ + return blobRepository; } /** diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java index 454633de5a1..0defdad363d 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginBag.java +++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java @@ -386,7 +386,7 @@ public class PluginBag implements AutoCloseable { */ public static class RuntimeLib implements PluginInfoInitialized, AutoCloseable { private String name, version, sig; - private JarRepository.JarContentRef jarContent; + private BlobRepository.BlobContentRef jarContent; private final CoreContainer coreContainer; private boolean verified = false; @@ -410,7 +410,7 @@ public class PluginBag implements AutoCloseable { if (jarContent != null) return; synchronized (this) { if (jarContent != null) return; - jarContent = coreContainer.getJarRepository().getJarIncRef(name + "/" + version); + jarContent = coreContainer.getBlobRepository().getBlobIncRef(name + "/" + version); } } @@ -430,13 +430,13 @@ public class PluginBag implements AutoCloseable { public ByteBuffer getFileContent(String entryName) throws IOException { if (jarContent == null) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jar not available: " + name + "/" + version); - return jarContent.jar.getFileContent(entryName); + return BlobRepository.getFileContent(jarContent.blob, entryName); } @Override public void close() throws Exception { - if (jarContent != null) coreContainer.getJarRepository().decrementJarRefCount(jarContent); + if (jarContent != null) coreContainer.getBlobRepository().decrementBlobRefCount(jarContent); } public static List getLibObjects(SolrCore core, List libs) { @@ -472,7 +472,7 @@ public class PluginBag implements AutoCloseable { } try { - String matchedKey = jarContent.jar.checkSignature(sig, new CryptoKeys(keys)); + String matchedKey = jarContent.blob.checkSignature(sig, new CryptoKeys(keys)); if (matchedKey == null) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No key matched signature for jar : " + name + " version: " + version); log.info("Jar {} signed with {} successfully verified", name, matchedKey);