From 170f5fb7a372fb78d181eec1fcb03cbd14f4a363 Mon Sep 17 00:00:00 2001 From: Andrzej Bialecki Date: Wed, 24 Apr 2019 12:01:05 +0200 Subject: [PATCH] SOLR-13423: Upgrade RRD4j to version 3.5. --- lucene/ivy-versions.properties | 2 +- solr/CHANGES.txt | 2 + .../handler/admin/MetricsHistoryHandler.java | 3 +- .../solr/metrics/rrd/SolrRrdBackend.java | 161 ++++++++++++++++-- .../metrics/rrd/SolrRrdBackendFactory.java | 22 ++- .../rrd/SolrRrdBackendFactoryTest.java | 2 + solr/licenses/rrd4j-3.2.jar.sha1 | 1 - solr/licenses/rrd4j-3.5.jar.sha1 | 1 + 8 files changed, 171 insertions(+), 23 deletions(-) delete mode 100644 solr/licenses/rrd4j-3.2.jar.sha1 create mode 100644 solr/licenses/rrd4j-3.5.jar.sha1 diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index d1994c21f97..98c94d729cb 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -285,7 +285,7 @@ org.restlet.jee.version = 2.3.0 /org.restlet.jee/org.restlet = ${org.restlet.jee.version} /org.restlet.jee/org.restlet.ext.servlet = ${org.restlet.jee.version} -/org.rrd4j/rrd4j = 3.2 +/org.rrd4j/rrd4j = 3.5 /org.simpleframework/simple-xml = 2.7.1 diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 77f27bf9316..80f637cff81 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -296,6 +296,8 @@ Other Changes * SOLR-13400: Replace Observable pattern in TransientSolrCoreCache (Erick Erickson) +* SOLR-13423: Upgrade RRD4j to version 3.5. (ab) + ================== 8.0.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java index bac0b6b9654..51bad597186 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java @@ -25,6 +25,7 @@ import java.io.Closeable; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; @@ -754,7 +755,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss } if (factory.exists(name)) { // get a throwaway copy (safe to close and discard) - RrdDb db = new RrdDb(URI_PREFIX + name, true, factory); + RrdDb db = RrdDb.getBuilder().setBackendFactory(factory).setReadOnly(true).setPath(new URI(URI_PREFIX + name)).build(); SimpleOrderedMap status = new SimpleOrderedMap<>(); status.add("status", getDbStatus(db)); status.add("node", nodeName); diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java index d0aa3e23566..643f315e831 100644 --- a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java +++ b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java @@ -56,7 +56,7 @@ public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable { try { SyncData syncData = factory.getData(path); if (syncData != null) { - this.buffer = syncData.data; + setBuffer(syncData.data); this.lastModifiedTime = syncData.timestamp; } } catch (IOException e) { @@ -75,9 +75,10 @@ public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable { readOnly = true; factory = null; this.lastModifiedTime = other.lastModifiedTime; - byte[] otherBuffer = other.buffer; - buffer = new byte[otherBuffer.length]; - System.arraycopy(otherBuffer, 0, buffer, 0, otherBuffer.length); + byte[] otherBuffer = other.getBuffer(); + byte[] newBuffer = new byte[otherBuffer.length]; + System.arraycopy(otherBuffer, 0, newBuffer, 0, otherBuffer.length); + super.setBuffer(newBuffer); } public boolean isReadOnly() { @@ -88,6 +89,11 @@ public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable { return lastModifiedTime; } + private void markDirty() { + lastModifiedTime = TimeUnit.MILLISECONDS.convert(factory.getTimeSource().getEpochTimeNs(), TimeUnit.NANOSECONDS); + dirty = true; + } + @Override protected void write(long offset, byte[] bytes) throws IOException { if (readOnly || closed) { @@ -96,14 +102,144 @@ public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable { lock.lock(); try { super.write(offset, bytes); - lastModifiedTime = TimeUnit.MILLISECONDS.convert(factory.getTimeSource().getEpochTimeNs(), TimeUnit.NANOSECONDS); - dirty = true; + markDirty(); } finally { lock.unlock(); } } - public SyncData getSyncData() { + @Override + protected void writeShort(long offset, short value) throws IOException { + if (readOnly || closed) { + return; + } + lock.lock(); + try { + super.writeShort(offset, value); + markDirty(); + } finally { + lock.unlock(); + } + } + + @Override + protected void writeInt(long offset, int value) throws IOException { + if (readOnly || closed) { + return; + } + lock.lock(); + try { + super.writeInt(offset, value); + markDirty(); + } finally { + lock.unlock(); + } + } + + @Override + protected void writeLong(long offset, long value) throws IOException { + if (readOnly || closed) { + return; + } + lock.lock(); + try { + super.writeLong(offset, value); + markDirty(); + } finally { + lock.unlock(); + } + } + + @Override + protected void writeDouble(long offset, double value) throws IOException { + if (readOnly || closed) { + return; + } + lock.lock(); + try { + super.writeDouble(offset, value); + markDirty(); + } finally { + lock.unlock(); + } + } + + @Override + protected void writeDouble(long offset, double value, int count) throws IOException { + if (readOnly || closed) { + return; + } + lock.lock(); + try { + super.writeDouble(offset, value, count); + markDirty(); + } finally { + lock.unlock(); + } + } + + @Override + protected void writeDouble(long offset, double[] values) throws IOException { + if (readOnly || closed) { + return; + } + lock.lock(); + try { + super.writeDouble(offset, values); + markDirty(); + } finally { + lock.unlock(); + } + } + + @Override + protected void writeString(long offset, String value, int length) throws IOException { + if (readOnly || closed) { + return; + } + lock.lock(); + try { + super.writeString(offset, value, length); + markDirty(); + } finally { + lock.unlock(); + } + } + + @Override + protected boolean isDirty() { + return dirty; + } + + @Override + protected void setBuffer(byte[] buffer) { + if (readOnly || closed) { + return; + } + lock.lock(); + try { + super.setBuffer(buffer); + markDirty(); + } finally { + lock.unlock(); + } + } + + @Override + protected void setLength(long length) throws IOException { + if (readOnly || closed) { + return; + } + lock.lock(); + try { + super.setLength(length); + markDirty(); + } finally { + lock.unlock(); + } + } + + public SyncData getSyncDataAndMarkClean() { if (readOnly || closed) { return null; } @@ -113,18 +249,17 @@ public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable { // hold a lock to block writes so that we get consistent data lock.lock(); try { - byte[] bufferCopy = new byte[buffer.length]; - System.arraycopy(buffer, 0, bufferCopy, 0, buffer.length); + byte[] oldBuffer = getBuffer(); + byte[] bufferCopy = new byte[oldBuffer.length]; + System.arraycopy(oldBuffer, 0, bufferCopy, 0, oldBuffer.length); return new SyncData(bufferCopy, lastModifiedTime); } finally { + // reset the dirty flag + dirty = false; lock.unlock(); } } - public void markClean() { - dirty = false; - } - @Override public void close() throws IOException { super.close(); diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java index 5448d8f79fe..ac128852d26 100644 --- a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java +++ b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java @@ -152,6 +152,20 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos } } + @Override + public URI getCanonicalUri(URI uri) { + return uri; + } + +// @Override +// protected URI getRootUri() { +// try { +// return new URI("solr", null, null, null); +// } catch (URISyntaxException e) { +// throw new RuntimeException("Impossible error", e); +// } +// } +// /** * Open (or get) a backend. * @param path backend path (without URI scheme) @@ -336,7 +350,7 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos log.debug("-- maybe sync backends: " + backends.keySet()); Map syncDatas = new HashMap<>(); backends.forEach((path, backend) -> { - SolrRrdBackend.SyncData syncData = backend.getSyncData(); + SolrRrdBackend.SyncData syncData = backend.getSyncDataAndMarkClean(); if (syncData != null) { syncDatas.put(backend.getPath(), syncData); } @@ -367,12 +381,6 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos } catch (SolrServerException e) { log.warn("Error committing RRD data updates", e); } - syncDatas.forEach((path, data) -> { - SolrRrdBackend backend = backends.get(path); - if (backend != null) { - backend.markClean(); - } - }); } catch (IOException e) { log.warn("Error sending RRD data updates", e); } diff --git a/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java b/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java index cdc1b619280..955c256f6b6 100644 --- a/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java @@ -27,6 +27,7 @@ import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.util.Pair; import org.apache.solr.common.util.TimeSource; import org.apache.solr.common.util.Utils; +import org.apache.solr.util.LogLevel; import org.apache.solr.util.MockSearchableSolrClient; import org.junit.After; import org.junit.Before; @@ -42,6 +43,7 @@ import org.rrd4j.core.Sample; /** * */ +@LogLevel("org.apache.solr.metrics.rrd=DEBUG") public class SolrRrdBackendFactoryTest extends SolrTestCaseJ4 { private SolrRrdBackendFactory factory; diff --git a/solr/licenses/rrd4j-3.2.jar.sha1 b/solr/licenses/rrd4j-3.2.jar.sha1 deleted file mode 100644 index 5304919414c..00000000000 --- a/solr/licenses/rrd4j-3.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8d480d5aa87b3d358862b78d6fa3660396220dc7 diff --git a/solr/licenses/rrd4j-3.5.jar.sha1 b/solr/licenses/rrd4j-3.5.jar.sha1 new file mode 100644 index 00000000000..8277f7ee829 --- /dev/null +++ b/solr/licenses/rrd4j-3.5.jar.sha1 @@ -0,0 +1 @@ +540c946b471dc915b0beb7c07069e3946665ef5d