HADOOP-8929. Add toString, other improvements for SampleQuantiles. Contributed by Todd Lipcon.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1398659 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
85bd8e0b72
commit
da255e41f4
|
@ -49,6 +49,8 @@ Release 2.0.3-alpha - Unreleased
|
|||
|
||||
HADOOP-8784. Improve IPC.Client's token use (daryn)
|
||||
|
||||
HADOOP-8929. Add toString, other improvements for SampleQuantiles (todd)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
|
||||
|
|
|
@ -20,7 +20,6 @@ package org.apache.hadoop.metrics2.lib;
|
|||
|
||||
import static org.apache.hadoop.metrics2.lib.Interns.info;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
|
@ -150,14 +149,8 @@ public class MutableQuantiles extends MutableMetric {
|
|||
@Override
|
||||
public void run() {
|
||||
synchronized (parent) {
|
||||
try {
|
||||
parent.previousCount = parent.estimator.getCount();
|
||||
parent.previousSnapshot = parent.estimator.snapshot();
|
||||
} catch (IOException e) {
|
||||
// Couldn't get a new snapshot because the window was empty
|
||||
parent.previousCount = 0;
|
||||
parent.previousSnapshot = null;
|
||||
}
|
||||
parent.previousCount = parent.estimator.getCount();
|
||||
parent.previousSnapshot = parent.estimator.snapshot();
|
||||
parent.estimator.clear();
|
||||
}
|
||||
parent.setChanged();
|
||||
|
|
|
@ -20,12 +20,14 @@ package org.apache.hadoop.metrics2.util;
|
|||
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
|
||||
import com.google.common.collect.ComparisonChain;
|
||||
|
||||
/**
|
||||
* Specifies a quantile (with error bounds) to be watched by a
|
||||
* {@link SampleQuantiles} object.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class Quantile {
|
||||
public class Quantile implements Comparable<Quantile> {
|
||||
public final double quantile;
|
||||
public final double error;
|
||||
|
||||
|
@ -57,4 +59,19 @@ public class Quantile {
|
|||
return (int) (Double.doubleToLongBits(quantile) ^ Double
|
||||
.doubleToLongBits(error));
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(Quantile other) {
|
||||
return ComparisonChain.start()
|
||||
.compare(quantile, other.quantile)
|
||||
.compare(error, other.error)
|
||||
.result();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.format("%.2f %%ile +/- %.2f%%",
|
||||
quantile * 100, error * 100);
|
||||
}
|
||||
|
||||
}
|
|
@ -18,16 +18,17 @@
|
|||
|
||||
package org.apache.hadoop.metrics2.util;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.ListIterator;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
/**
|
||||
* Implementation of the Cormode, Korn, Muthukrishnan, and Srivastava algorithm
|
||||
|
@ -202,10 +203,8 @@ public class SampleQuantiles {
|
|||
* @param quantile Queried quantile, e.g. 0.50 or 0.99.
|
||||
* @return Estimated value at that quantile.
|
||||
*/
|
||||
private long query(double quantile) throws IOException {
|
||||
if (samples.size() == 0) {
|
||||
throw new IOException("No samples present");
|
||||
}
|
||||
private long query(double quantile) {
|
||||
Preconditions.checkState(!samples.isEmpty(), "no data in estimator");
|
||||
|
||||
int rankMin = 0;
|
||||
int desired = (int) (quantile * count);
|
||||
|
@ -231,14 +230,18 @@ public class SampleQuantiles {
|
|||
/**
|
||||
* Get a snapshot of the current values of all the tracked quantiles.
|
||||
*
|
||||
* @return snapshot of the tracked quantiles
|
||||
* @throws IOException
|
||||
* if no items have been added to the estimator
|
||||
* @return snapshot of the tracked quantiles. If no items are added
|
||||
* to the estimator, returns null.
|
||||
*/
|
||||
synchronized public Map<Quantile, Long> snapshot() throws IOException {
|
||||
synchronized public Map<Quantile, Long> snapshot() {
|
||||
// flush the buffer first for best results
|
||||
insertBatch();
|
||||
Map<Quantile, Long> values = new HashMap<Quantile, Long>(quantiles.length);
|
||||
|
||||
if (samples.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Map<Quantile, Long> values = new TreeMap<Quantile, Long>();
|
||||
for (int i = 0; i < quantiles.length; i++) {
|
||||
values.put(quantiles[i], query(quantiles[i].quantile));
|
||||
}
|
||||
|
@ -273,6 +276,16 @@ public class SampleQuantiles {
|
|||
bufferCount = 0;
|
||||
samples.clear();
|
||||
}
|
||||
|
||||
@Override
|
||||
synchronized public String toString() {
|
||||
Map<Quantile, Long> data = snapshot();
|
||||
if (data == null) {
|
||||
return "[no samples]";
|
||||
} else {
|
||||
return Joiner.on("\n").withKeyValueSeparator(": ").join(data);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Describes a measured value passed to the estimator, tracking additional
|
||||
|
|
|
@ -18,9 +18,7 @@
|
|||
|
||||
package org.apache.hadoop.metrics2.util;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
|
@ -28,7 +26,6 @@ import java.util.Collections;
|
|||
import java.util.Map;
|
||||
import java.util.Random;
|
||||
|
||||
import org.apache.hadoop.test.GenericTestUtils;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
|
@ -54,18 +51,22 @@ public class TestSampleQuantiles {
|
|||
// Counts start off zero
|
||||
assertEquals(estimator.getCount(), 0);
|
||||
assertEquals(estimator.getSampleCount(), 0);
|
||||
try {
|
||||
estimator.snapshot();
|
||||
fail("Expected IOException from empty window");
|
||||
} catch (IOException e) {
|
||||
GenericTestUtils.assertExceptionContains("No samples", e);
|
||||
}
|
||||
|
||||
// Snapshot should be null if there are no entries.
|
||||
assertNull(estimator.snapshot());
|
||||
|
||||
// Count increment correctly by 1
|
||||
estimator.insert(1337);
|
||||
assertEquals(estimator.getCount(), 1);
|
||||
estimator.snapshot();
|
||||
assertEquals(estimator.getSampleCount(), 1);
|
||||
|
||||
assertEquals(
|
||||
"50.00 %ile +/- 5.00%: 1337\n" +
|
||||
"75.00 %ile +/- 2.50%: 1337\n" +
|
||||
"90.00 %ile +/- 1.00%: 1337\n" +
|
||||
"95.00 %ile +/- 0.50%: 1337\n" +
|
||||
"99.00 %ile +/- 0.10%: 1337", estimator.toString());
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -80,12 +81,7 @@ public class TestSampleQuantiles {
|
|||
estimator.clear();
|
||||
assertEquals(estimator.getCount(), 0);
|
||||
assertEquals(estimator.getSampleCount(), 0);
|
||||
try {
|
||||
estimator.snapshot();
|
||||
fail("Expected IOException for an empty window.");
|
||||
} catch (IOException e) {
|
||||
GenericTestUtils.assertExceptionContains("No samples", e);
|
||||
}
|
||||
assertNull(estimator.snapshot());
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
Loading…
Reference in New Issue