HDFS-13149. Ozone: Rename Corona to Freon. Contributed by Anu Engineer.
This commit is contained in:
parent
cd807c9890
commit
136b419594
|
@ -46,6 +46,7 @@ function hadoop_usage
|
||||||
hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables"
|
hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables"
|
||||||
hadoop_add_subcommand "ec" admin "run a HDFS ErasureCoding CLI"
|
hadoop_add_subcommand "ec" admin "run a HDFS ErasureCoding CLI"
|
||||||
hadoop_add_subcommand "fetchdt" client "fetch a delegation token from the NameNode"
|
hadoop_add_subcommand "fetchdt" client "fetch a delegation token from the NameNode"
|
||||||
|
hadoop_add_subcommand "freon" client "runs an ozone data generator"
|
||||||
hadoop_add_subcommand "fsck" admin "run a DFS filesystem checking utility"
|
hadoop_add_subcommand "fsck" admin "run a DFS filesystem checking utility"
|
||||||
hadoop_add_subcommand "getconf" client "get config values from configuration"
|
hadoop_add_subcommand "getconf" client "get config values from configuration"
|
||||||
hadoop_add_subcommand "groups" client "get the groups which users belong to"
|
hadoop_add_subcommand "groups" client "get the groups which users belong to"
|
||||||
|
@ -139,6 +140,9 @@ function hdfscmd_case
|
||||||
fetchdt)
|
fetchdt)
|
||||||
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
|
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
|
||||||
;;
|
;;
|
||||||
|
freon)
|
||||||
|
HADOOP_CLASSNAME=org.apache.hadoop.ozone.tools.Freon
|
||||||
|
;;
|
||||||
fsck)
|
fsck)
|
||||||
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSck
|
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSck
|
||||||
;;
|
;;
|
||||||
|
|
|
@ -62,7 +62,7 @@ import java.util.function.Supplier;
|
||||||
import static java.lang.Math.min;
|
import static java.lang.Math.min;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Corona - A tool to populate ozone with data for testing.<br>
|
* Freon - A tool to populate ozone with data for testing.<br>
|
||||||
* This is not a map-reduce program and this is not for benchmarking
|
* This is not a map-reduce program and this is not for benchmarking
|
||||||
* Ozone write throughput.<br>
|
* Ozone write throughput.<br>
|
||||||
* It supports both online and offline modes. Default mode is offline,
|
* It supports both online and offline modes. Default mode is offline,
|
||||||
|
@ -92,9 +92,9 @@ import static java.lang.Math.min;
|
||||||
* can be used to override</li>
|
* can be used to override</li>
|
||||||
* </ul>
|
* </ul>
|
||||||
*/
|
*/
|
||||||
public final class Corona extends Configured implements Tool {
|
public final class Freon extends Configured implements Tool {
|
||||||
|
|
||||||
enum CoronaOps {
|
enum FreonOps {
|
||||||
VOLUME_CREATE,
|
VOLUME_CREATE,
|
||||||
BUCKET_CREATE,
|
BUCKET_CREATE,
|
||||||
KEY_CREATE,
|
KEY_CREATE,
|
||||||
|
@ -127,7 +127,7 @@ public final class Corona extends Configured implements Tool {
|
||||||
private static final int QUANTILES = 10;
|
private static final int QUANTILES = 10;
|
||||||
|
|
||||||
private static final Logger LOG =
|
private static final Logger LOG =
|
||||||
LoggerFactory.getLogger(Corona.class);
|
LoggerFactory.getLogger(Freon.class);
|
||||||
|
|
||||||
private boolean printUsage = false;
|
private boolean printUsage = false;
|
||||||
private boolean completed = false;
|
private boolean completed = false;
|
||||||
|
@ -176,7 +176,7 @@ public final class Corona extends Configured implements Tool {
|
||||||
private ArrayList<Histogram> histograms = new ArrayList<>();
|
private ArrayList<Histogram> histograms = new ArrayList<>();
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
Corona(Configuration conf) throws IOException {
|
Freon(Configuration conf) throws IOException {
|
||||||
startTime = System.nanoTime();
|
startTime = System.nanoTime();
|
||||||
jobStartTime = System.currentTimeMillis();
|
jobStartTime = System.currentTimeMillis();
|
||||||
volumeCreationTime = new AtomicLong();
|
volumeCreationTime = new AtomicLong();
|
||||||
|
@ -189,7 +189,7 @@ public final class Corona extends Configured implements Tool {
|
||||||
numberOfKeysAdded = new AtomicLong();
|
numberOfKeysAdded = new AtomicLong();
|
||||||
ozoneClient = OzoneClientFactory.getClient(conf);
|
ozoneClient = OzoneClientFactory.getClient(conf);
|
||||||
objectStore = ozoneClient.getObjectStore();
|
objectStore = ozoneClient.getObjectStore();
|
||||||
for (CoronaOps ops : CoronaOps.values()) {
|
for (FreonOps ops : FreonOps.values()) {
|
||||||
histograms.add(ops.ordinal(), new Histogram(new UniformReservoir()));
|
histograms.add(ops.ordinal(), new Histogram(new UniformReservoir()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -199,7 +199,7 @@ public final class Corona extends Configured implements Tool {
|
||||||
*/
|
*/
|
||||||
public static void main(String[] args) throws Exception {
|
public static void main(String[] args) throws Exception {
|
||||||
Configuration conf = new OzoneConfiguration();
|
Configuration conf = new OzoneConfiguration();
|
||||||
int res = ToolRunner.run(conf, new Corona(conf), args);
|
int res = ToolRunner.run(conf, new Freon(conf), args);
|
||||||
System.exit(res);
|
System.exit(res);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -271,7 +271,7 @@ public final class Corona extends Configured implements Tool {
|
||||||
OptionBuilder.withArgName("online | offline");
|
OptionBuilder.withArgName("online | offline");
|
||||||
OptionBuilder.hasArg();
|
OptionBuilder.hasArg();
|
||||||
OptionBuilder.withDescription("specifies the mode of " +
|
OptionBuilder.withDescription("specifies the mode of " +
|
||||||
"Corona run.");
|
"Freon run.");
|
||||||
Option optMode = OptionBuilder.create(MODE);
|
Option optMode = OptionBuilder.create(MODE);
|
||||||
|
|
||||||
OptionBuilder.withArgName("source url");
|
OptionBuilder.withArgName("source url");
|
||||||
|
@ -400,7 +400,7 @@ public final class Corona extends Configured implements Tool {
|
||||||
System.out.println("-jsonDir "
|
System.out.println("-jsonDir "
|
||||||
+ "directory where json is created.");
|
+ "directory where json is created.");
|
||||||
System.out.println("-mode [online | offline] "
|
System.out.println("-mode [online | offline] "
|
||||||
+ "specifies the mode in which Corona should run.");
|
+ "specifies the mode in which Freon should run.");
|
||||||
System.out.println("-source <url> "
|
System.out.println("-source <url> "
|
||||||
+ "specifies the URL of s3 commoncrawl warc file to " +
|
+ "specifies the URL of s3 commoncrawl warc file to " +
|
||||||
"be used when the mode is online.");
|
"be used when the mode is online.");
|
||||||
|
@ -446,7 +446,7 @@ public final class Corona extends Configured implements Tool {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Prints stats of {@link Corona} run to the PrintStream.
|
* Prints stats of {@link Freon} run to the PrintStream.
|
||||||
*
|
*
|
||||||
* @param out PrintStream
|
* @param out PrintStream
|
||||||
*/
|
*/
|
||||||
|
@ -512,10 +512,10 @@ public final class Corona extends Configured implements Tool {
|
||||||
if (jsonDir != null) {
|
if (jsonDir != null) {
|
||||||
|
|
||||||
String[][] quantileTime =
|
String[][] quantileTime =
|
||||||
new String[CoronaOps.values().length][QUANTILES + 1];
|
new String[FreonOps.values().length][QUANTILES + 1];
|
||||||
String[] deviations = new String[CoronaOps.values().length];
|
String[] deviations = new String[FreonOps.values().length];
|
||||||
String[] means = new String[CoronaOps.values().length];
|
String[] means = new String[FreonOps.values().length];
|
||||||
for (CoronaOps ops : CoronaOps.values()) {
|
for (FreonOps ops : FreonOps.values()) {
|
||||||
Snapshot snapshot = histograms.get(ops.ordinal()).getSnapshot();
|
Snapshot snapshot = histograms.get(ops.ordinal()).getSnapshot();
|
||||||
for (int i = 0; i <= QUANTILES; i++) {
|
for (int i = 0; i <= QUANTILES; i++) {
|
||||||
quantileTime[ops.ordinal()][i] = DurationFormatUtils.formatDuration(
|
quantileTime[ops.ordinal()][i] = DurationFormatUtils.formatDuration(
|
||||||
|
@ -531,26 +531,26 @@ public final class Corona extends Configured implements Tool {
|
||||||
DURATION_FORMAT);
|
DURATION_FORMAT);
|
||||||
}
|
}
|
||||||
|
|
||||||
CoronaJobInfo jobInfo = new CoronaJobInfo().setExecTime(execTime)
|
FreonJobInfo jobInfo = new FreonJobInfo().setExecTime(execTime)
|
||||||
.setGitBaseRevision(VersionInfo.getRevision())
|
.setGitBaseRevision(VersionInfo.getRevision())
|
||||||
.setMeanVolumeCreateTime(means[CoronaOps.VOLUME_CREATE.ordinal()])
|
.setMeanVolumeCreateTime(means[FreonOps.VOLUME_CREATE.ordinal()])
|
||||||
.setDeviationVolumeCreateTime(
|
.setDeviationVolumeCreateTime(
|
||||||
deviations[CoronaOps.VOLUME_CREATE.ordinal()])
|
deviations[FreonOps.VOLUME_CREATE.ordinal()])
|
||||||
.setTenQuantileVolumeCreateTime(
|
.setTenQuantileVolumeCreateTime(
|
||||||
quantileTime[CoronaOps.VOLUME_CREATE.ordinal()])
|
quantileTime[FreonOps.VOLUME_CREATE.ordinal()])
|
||||||
.setMeanBucketCreateTime(means[CoronaOps.BUCKET_CREATE.ordinal()])
|
.setMeanBucketCreateTime(means[FreonOps.BUCKET_CREATE.ordinal()])
|
||||||
.setDeviationBucketCreateTime(
|
.setDeviationBucketCreateTime(
|
||||||
deviations[CoronaOps.BUCKET_CREATE.ordinal()])
|
deviations[FreonOps.BUCKET_CREATE.ordinal()])
|
||||||
.setTenQuantileBucketCreateTime(
|
.setTenQuantileBucketCreateTime(
|
||||||
quantileTime[CoronaOps.BUCKET_CREATE.ordinal()])
|
quantileTime[FreonOps.BUCKET_CREATE.ordinal()])
|
||||||
.setMeanKeyCreateTime(means[CoronaOps.KEY_CREATE.ordinal()])
|
.setMeanKeyCreateTime(means[FreonOps.KEY_CREATE.ordinal()])
|
||||||
.setDeviationKeyCreateTime(deviations[CoronaOps.KEY_CREATE.ordinal()])
|
.setDeviationKeyCreateTime(deviations[FreonOps.KEY_CREATE.ordinal()])
|
||||||
.setTenQuantileKeyCreateTime(
|
.setTenQuantileKeyCreateTime(
|
||||||
quantileTime[CoronaOps.KEY_CREATE.ordinal()])
|
quantileTime[FreonOps.KEY_CREATE.ordinal()])
|
||||||
.setMeanKeyWriteTime(means[CoronaOps.KEY_WRITE.ordinal()])
|
.setMeanKeyWriteTime(means[FreonOps.KEY_WRITE.ordinal()])
|
||||||
.setDeviationKeyWriteTime(deviations[CoronaOps.KEY_WRITE.ordinal()])
|
.setDeviationKeyWriteTime(deviations[FreonOps.KEY_WRITE.ordinal()])
|
||||||
.setTenQuantileKeyWriteTime(
|
.setTenQuantileKeyWriteTime(
|
||||||
quantileTime[CoronaOps.KEY_WRITE.ordinal()]);
|
quantileTime[FreonOps.KEY_WRITE.ordinal()]);
|
||||||
String jsonName =
|
String jsonName =
|
||||||
new SimpleDateFormat("yyyyMMddHHmmss").format(Time.now()) + ".json";
|
new SimpleDateFormat("yyyyMMddHHmmss").format(Time.now()) + ".json";
|
||||||
String jsonPath = jsonDir + "/" + jsonName;
|
String jsonPath = jsonDir + "/" + jsonName;
|
||||||
|
@ -704,7 +704,7 @@ public final class Corona extends Configured implements Tool {
|
||||||
objectStore.createVolume(volumeName);
|
objectStore.createVolume(volumeName);
|
||||||
long volumeCreationDuration = System.nanoTime() - start;
|
long volumeCreationDuration = System.nanoTime() - start;
|
||||||
volumeCreationTime.getAndAdd(volumeCreationDuration);
|
volumeCreationTime.getAndAdd(volumeCreationDuration);
|
||||||
histograms.get(CoronaOps.VOLUME_CREATE.ordinal())
|
histograms.get(FreonOps.VOLUME_CREATE.ordinal())
|
||||||
.update(volumeCreationDuration);
|
.update(volumeCreationDuration);
|
||||||
numberOfVolumesCreated.getAndIncrement();
|
numberOfVolumesCreated.getAndIncrement();
|
||||||
volume = objectStore.getVolume(volumeName);
|
volume = objectStore.getVolume(volumeName);
|
||||||
|
@ -724,7 +724,7 @@ public final class Corona extends Configured implements Tool {
|
||||||
start = System.nanoTime();
|
start = System.nanoTime();
|
||||||
volume.createBucket(bucketName);
|
volume.createBucket(bucketName);
|
||||||
long bucketCreationDuration = System.nanoTime() - start;
|
long bucketCreationDuration = System.nanoTime() - start;
|
||||||
histograms.get(CoronaOps.BUCKET_CREATE.ordinal())
|
histograms.get(FreonOps.BUCKET_CREATE.ordinal())
|
||||||
.update(bucketCreationDuration);
|
.update(bucketCreationDuration);
|
||||||
bucketCreationTime.getAndAdd(bucketCreationDuration);
|
bucketCreationTime.getAndAdd(bucketCreationDuration);
|
||||||
numberOfBucketsCreated.getAndIncrement();
|
numberOfBucketsCreated.getAndIncrement();
|
||||||
|
@ -741,7 +741,7 @@ public final class Corona extends Configured implements Tool {
|
||||||
OzoneOutputStream os =
|
OzoneOutputStream os =
|
||||||
bucket.createKey(key, keySize, type, factor);
|
bucket.createKey(key, keySize, type, factor);
|
||||||
long keyCreationDuration = System.nanoTime() - keyCreateStart;
|
long keyCreationDuration = System.nanoTime() - keyCreateStart;
|
||||||
histograms.get(CoronaOps.KEY_CREATE.ordinal())
|
histograms.get(FreonOps.KEY_CREATE.ordinal())
|
||||||
.update(keyCreationDuration);
|
.update(keyCreationDuration);
|
||||||
keyCreationTime.getAndAdd(keyCreationDuration);
|
keyCreationTime.getAndAdd(keyCreationDuration);
|
||||||
long keyWriteStart = System.nanoTime();
|
long keyWriteStart = System.nanoTime();
|
||||||
|
@ -750,7 +750,7 @@ public final class Corona extends Configured implements Tool {
|
||||||
os.close();
|
os.close();
|
||||||
long keyWriteDuration = System.nanoTime() - keyWriteStart;
|
long keyWriteDuration = System.nanoTime() - keyWriteStart;
|
||||||
threadKeyWriteTime += keyWriteDuration;
|
threadKeyWriteTime += keyWriteDuration;
|
||||||
histograms.get(CoronaOps.KEY_WRITE.ordinal())
|
histograms.get(FreonOps.KEY_WRITE.ordinal())
|
||||||
.update(keyWriteDuration);
|
.update(keyWriteDuration);
|
||||||
totalBytesWritten.getAndAdd(keySize);
|
totalBytesWritten.getAndAdd(keySize);
|
||||||
numberOfKeysAdded.getAndIncrement();
|
numberOfKeysAdded.getAndIncrement();
|
||||||
|
@ -780,7 +780,7 @@ public final class Corona extends Configured implements Tool {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private final class CoronaJobInfo {
|
private final class FreonJobInfo {
|
||||||
|
|
||||||
private String status;
|
private String status;
|
||||||
private String gitBaseRevision;
|
private String gitBaseRevision;
|
||||||
|
@ -815,17 +815,17 @@ public final class Corona extends Configured implements Tool {
|
||||||
private String deviationKeyWriteTime;
|
private String deviationKeyWriteTime;
|
||||||
private String[] tenQuantileKeyWriteTime;
|
private String[] tenQuantileKeyWriteTime;
|
||||||
|
|
||||||
private CoronaJobInfo() {
|
private FreonJobInfo() {
|
||||||
this.status = exception ? "Failed" : "Success";
|
this.status = exception ? "Failed" : "Success";
|
||||||
this.numOfVolumes = Corona.this.numOfVolumes;
|
this.numOfVolumes = Freon.this.numOfVolumes;
|
||||||
this.numOfBuckets = Corona.this.numOfBuckets;
|
this.numOfBuckets = Freon.this.numOfBuckets;
|
||||||
this.numOfKeys = Corona.this.numOfKeys;
|
this.numOfKeys = Freon.this.numOfKeys;
|
||||||
this.numOfThreads = Corona.this.numOfThreads;
|
this.numOfThreads = Freon.this.numOfThreads;
|
||||||
this.keySize = Corona.this.keySize;
|
this.keySize = Freon.this.keySize;
|
||||||
this.mode = Corona.this.mode;
|
this.mode = Freon.this.mode;
|
||||||
this.jobStartTime = Time.formatTime(Corona.this.jobStartTime);
|
this.jobStartTime = Time.formatTime(Freon.this.jobStartTime);
|
||||||
this.replicationFactor = Corona.this.factor.name();
|
this.replicationFactor = Freon.this.factor.name();
|
||||||
this.replicationType = Corona.this.type.name();
|
this.replicationType = Freon.this.type.name();
|
||||||
|
|
||||||
long totalBytes =
|
long totalBytes =
|
||||||
Long.parseLong(numOfVolumes) * Long.parseLong(numOfBuckets) * Long
|
Long.parseLong(numOfVolumes) * Long.parseLong(numOfBuckets) * Long
|
||||||
|
@ -833,7 +833,7 @@ public final class Corona extends Configured implements Tool {
|
||||||
this.dataWritten = getInStorageUnits((double) totalBytes);
|
this.dataWritten = getInStorageUnits((double) totalBytes);
|
||||||
this.totalThroughputPerSecond = getInStorageUnits(
|
this.totalThroughputPerSecond = getInStorageUnits(
|
||||||
(totalBytes * 1.0) / TimeUnit.NANOSECONDS
|
(totalBytes * 1.0) / TimeUnit.NANOSECONDS
|
||||||
.toSeconds(Corona.this.keyWriteTime.get() / threadPoolSize));
|
.toSeconds(Freon.this.keyWriteTime.get() / threadPoolSize));
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getInStorageUnits(Double value) {
|
private String getInStorageUnits(Double value) {
|
||||||
|
@ -858,81 +858,81 @@ public final class Corona extends Configured implements Tool {
|
||||||
return size + " " + unit;
|
return size + " " + unit;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CoronaJobInfo setGitBaseRevision(String gitBaseRevisionVal) {
|
public FreonJobInfo setGitBaseRevision(String gitBaseRevisionVal) {
|
||||||
gitBaseRevision = gitBaseRevisionVal;
|
gitBaseRevision = gitBaseRevisionVal;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CoronaJobInfo setExecTime(String execTimeVal) {
|
public FreonJobInfo setExecTime(String execTimeVal) {
|
||||||
execTime = execTimeVal;
|
execTime = execTimeVal;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CoronaJobInfo setMeanKeyWriteTime(String deviationKeyWriteTimeVal) {
|
public FreonJobInfo setMeanKeyWriteTime(String deviationKeyWriteTimeVal) {
|
||||||
this.meanKeyWriteTime = deviationKeyWriteTimeVal;
|
this.meanKeyWriteTime = deviationKeyWriteTimeVal;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CoronaJobInfo setDeviationKeyWriteTime(
|
public FreonJobInfo setDeviationKeyWriteTime(
|
||||||
String deviationKeyWriteTimeVal) {
|
String deviationKeyWriteTimeVal) {
|
||||||
this.deviationKeyWriteTime = deviationKeyWriteTimeVal;
|
this.deviationKeyWriteTime = deviationKeyWriteTimeVal;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CoronaJobInfo setTenQuantileKeyWriteTime(
|
public FreonJobInfo setTenQuantileKeyWriteTime(
|
||||||
String[] tenQuantileKeyWriteTimeVal) {
|
String[] tenQuantileKeyWriteTimeVal) {
|
||||||
this.tenQuantileKeyWriteTime = tenQuantileKeyWriteTimeVal;
|
this.tenQuantileKeyWriteTime = tenQuantileKeyWriteTimeVal;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CoronaJobInfo setMeanKeyCreateTime(String deviationKeyWriteTimeVal) {
|
public FreonJobInfo setMeanKeyCreateTime(String deviationKeyWriteTimeVal) {
|
||||||
this.meanKeyCreateTime = deviationKeyWriteTimeVal;
|
this.meanKeyCreateTime = deviationKeyWriteTimeVal;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CoronaJobInfo setDeviationKeyCreateTime(
|
public FreonJobInfo setDeviationKeyCreateTime(
|
||||||
String deviationKeyCreateTimeVal) {
|
String deviationKeyCreateTimeVal) {
|
||||||
this.deviationKeyCreateTime = deviationKeyCreateTimeVal;
|
this.deviationKeyCreateTime = deviationKeyCreateTimeVal;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CoronaJobInfo setTenQuantileKeyCreateTime(
|
public FreonJobInfo setTenQuantileKeyCreateTime(
|
||||||
String[] tenQuantileKeyCreateTimeVal) {
|
String[] tenQuantileKeyCreateTimeVal) {
|
||||||
this.tenQuantileKeyCreateTime = tenQuantileKeyCreateTimeVal;
|
this.tenQuantileKeyCreateTime = tenQuantileKeyCreateTimeVal;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CoronaJobInfo setMeanBucketCreateTime(
|
public FreonJobInfo setMeanBucketCreateTime(
|
||||||
String deviationKeyWriteTimeVal) {
|
String deviationKeyWriteTimeVal) {
|
||||||
this.meanBucketCreateTime = deviationKeyWriteTimeVal;
|
this.meanBucketCreateTime = deviationKeyWriteTimeVal;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CoronaJobInfo setDeviationBucketCreateTime(
|
public FreonJobInfo setDeviationBucketCreateTime(
|
||||||
String deviationBucketCreateTimeVal) {
|
String deviationBucketCreateTimeVal) {
|
||||||
this.deviationBucketCreateTime = deviationBucketCreateTimeVal;
|
this.deviationBucketCreateTime = deviationBucketCreateTimeVal;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CoronaJobInfo setTenQuantileBucketCreateTime(
|
public FreonJobInfo setTenQuantileBucketCreateTime(
|
||||||
String[] tenQuantileBucketCreateTimeVal) {
|
String[] tenQuantileBucketCreateTimeVal) {
|
||||||
this.tenQuantileBucketCreateTime = tenQuantileBucketCreateTimeVal;
|
this.tenQuantileBucketCreateTime = tenQuantileBucketCreateTimeVal;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CoronaJobInfo setMeanVolumeCreateTime(
|
public FreonJobInfo setMeanVolumeCreateTime(
|
||||||
String deviationKeyWriteTimeVal) {
|
String deviationKeyWriteTimeVal) {
|
||||||
this.meanVolumeCreateTime = deviationKeyWriteTimeVal;
|
this.meanVolumeCreateTime = deviationKeyWriteTimeVal;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CoronaJobInfo setDeviationVolumeCreateTime(
|
public FreonJobInfo setDeviationVolumeCreateTime(
|
||||||
String deviationVolumeCreateTimeVal) {
|
String deviationVolumeCreateTimeVal) {
|
||||||
this.deviationVolumeCreateTime = deviationVolumeCreateTimeVal;
|
this.deviationVolumeCreateTime = deviationVolumeCreateTimeVal;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CoronaJobInfo setTenQuantileVolumeCreateTime(
|
public FreonJobInfo setTenQuantileVolumeCreateTime(
|
||||||
String[] tenQuantileVolumeCreateTimeVal) {
|
String[] tenQuantileVolumeCreateTimeVal) {
|
||||||
this.tenQuantileVolumeCreateTime = tenQuantileVolumeCreateTimeVal;
|
this.tenQuantileVolumeCreateTime = tenQuantileVolumeCreateTimeVal;
|
||||||
return this;
|
return this;
|
|
@ -61,18 +61,18 @@ the background.
|
||||||
|
|
||||||
- `docker-compose up -d`
|
- `docker-compose up -d`
|
||||||
|
|
||||||
Now let us run some work load against ozone, to do that we will run corona.
|
Now let us run some work load against ozone, to do that we will run freon.
|
||||||
|
|
||||||
This will log into the datanode and run bash.
|
This will log into the datanode and run bash.
|
||||||
|
|
||||||
- `docker-compose exec datanode bash`
|
- `docker-compose exec datanode bash`
|
||||||
- `cd hadoop/bin`
|
- `cd hadoop/bin`
|
||||||
|
|
||||||
Now you can run the oz command shell or corona the ozone load generator.
|
Now you can run the oz command shell or freon, the ozone load generator.
|
||||||
|
|
||||||
This is the command to run corona.
|
This is the command to run freon.
|
||||||
|
|
||||||
- `./hdfs corona -mode offline -validateWrites -numOfVolumes 1 -numOfBuckets 10 -numOfKeys 100`
|
- `./hdfs freon -mode offline -validateWrites -numOfVolumes 1 -numOfBuckets 10 -numOfKeys 100`
|
||||||
|
|
||||||
You can checkout the KSM UI to see the requests information.
|
You can checkout the KSM UI to see the requests information.
|
||||||
|
|
||||||
|
|
|
@ -22,11 +22,11 @@ keys.
The following is a high-level overview of the core components of Ozone.
|
||||||
The main elements of Ozone are
:
|
The main elements of Ozone are
:
|
||||||
|
|
||||||
### Clients
|
### Clients
|
||||||
Ozone ships with a set of ready-made clients. They are
Ozone CLI and Corona.
|
Ozone ships with a set of ready-made clients. They are
Ozone CLI and Freon.
|
||||||
|
|
||||||
* [Ozone CLI](./OzoneCommandShell.html) is the command line interface like 'hdfs' command.
|
* [Ozone CLI](./OzoneCommandShell.html) is the command line interface like 'hdfs' command.
|
||||||
|
|
||||||
* Corona is a load generation tool for Ozone.
|
* Freon is a load generation tool for Ozone.
|
||||||
|
|
||||||
### REST Handler
|
### REST Handler
|
||||||
Ozone provides both an RPC (Remote Procedure Call) as well as a REST
|
Ozone provides both an RPC (Remote Procedure Call) as well as a REST
|
||||||
|
|
|
@ -36,9 +36,9 @@ import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tests Corona, with MiniOzoneCluster.
|
* Tests Freon, with MiniOzoneCluster.
|
||||||
*/
|
*/
|
||||||
public class TestCorona {
|
public class TestFreon {
|
||||||
|
|
||||||
private static MiniOzoneCluster cluster;
|
private static MiniOzoneCluster cluster;
|
||||||
private static OzoneConfiguration conf;
|
private static OzoneConfiguration conf;
|
||||||
|
@ -80,13 +80,13 @@ public class TestCorona {
|
||||||
args.add("5");
|
args.add("5");
|
||||||
args.add("-numOfKeys");
|
args.add("-numOfKeys");
|
||||||
args.add("10");
|
args.add("10");
|
||||||
Corona corona = new Corona(conf);
|
Freon freon = new Freon(conf);
|
||||||
int res = ToolRunner.run(conf, corona,
|
int res = ToolRunner.run(conf, freon,
|
||||||
args.toArray(new String[0]));
|
args.toArray(new String[0]));
|
||||||
Assert.assertEquals(2, corona.getNumberOfVolumesCreated());
|
Assert.assertEquals(2, freon.getNumberOfVolumesCreated());
|
||||||
Assert.assertEquals(10, corona.getNumberOfBucketsCreated());
|
Assert.assertEquals(10, freon.getNumberOfBucketsCreated());
|
||||||
Assert.assertEquals(100, corona.getNumberOfKeysAdded());
|
Assert.assertEquals(100, freon.getNumberOfKeysAdded());
|
||||||
Assert.assertEquals(10240 - 36, corona.getKeyValueLength());
|
Assert.assertEquals(10240 - 36, freon.getKeyValueLength());
|
||||||
Assert.assertEquals(0, res);
|
Assert.assertEquals(0, res);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -103,17 +103,17 @@ public class TestCorona {
|
||||||
args.add("5");
|
args.add("5");
|
||||||
args.add("-numOfKeys");
|
args.add("-numOfKeys");
|
||||||
args.add("10");
|
args.add("10");
|
||||||
Corona corona = new Corona(conf);
|
Freon freon = new Freon(conf);
|
||||||
int res = ToolRunner.run(conf, corona,
|
int res = ToolRunner.run(conf, freon,
|
||||||
args.toArray(new String[0]));
|
args.toArray(new String[0]));
|
||||||
Assert.assertEquals(0, res);
|
Assert.assertEquals(0, res);
|
||||||
Assert.assertEquals(2, corona.getNumberOfVolumesCreated());
|
Assert.assertEquals(2, freon.getNumberOfVolumesCreated());
|
||||||
Assert.assertEquals(10, corona.getNumberOfBucketsCreated());
|
Assert.assertEquals(10, freon.getNumberOfBucketsCreated());
|
||||||
Assert.assertEquals(100, corona.getNumberOfKeysAdded());
|
Assert.assertEquals(100, freon.getNumberOfKeysAdded());
|
||||||
Assert.assertTrue(corona.getValidateWrites());
|
Assert.assertTrue(freon.getValidateWrites());
|
||||||
Assert.assertNotEquals(0, corona.getTotalKeysValidated());
|
Assert.assertNotEquals(0, freon.getTotalKeysValidated());
|
||||||
Assert.assertNotEquals(0, corona.getSuccessfulValidationCount());
|
Assert.assertNotEquals(0, freon.getSuccessfulValidationCount());
|
||||||
Assert.assertEquals(0, corona.getUnsuccessfulValidationCount());
|
Assert.assertEquals(0, freon.getUnsuccessfulValidationCount());
|
||||||
System.setOut(originalStream);
|
System.setOut(originalStream);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -130,12 +130,12 @@ public class TestCorona {
|
||||||
args.add("10");
|
args.add("10");
|
||||||
args.add("-keySize");
|
args.add("-keySize");
|
||||||
args.add("10240");
|
args.add("10240");
|
||||||
Corona corona = new Corona(conf);
|
Freon freon = new Freon(conf);
|
||||||
int res = ToolRunner.run(conf, corona,
|
int res = ToolRunner.run(conf, freon,
|
||||||
args.toArray(new String[0]));
|
args.toArray(new String[0]));
|
||||||
Assert.assertEquals(10, corona.getNumberOfVolumesCreated());
|
Assert.assertEquals(10, freon.getNumberOfVolumesCreated());
|
||||||
Assert.assertEquals(10, corona.getNumberOfBucketsCreated());
|
Assert.assertEquals(10, freon.getNumberOfBucketsCreated());
|
||||||
Assert.assertEquals(100, corona.getNumberOfKeysAdded());
|
Assert.assertEquals(100, freon.getNumberOfKeysAdded());
|
||||||
Assert.assertEquals(0, res);
|
Assert.assertEquals(0, res);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -154,12 +154,12 @@ public class TestCorona {
|
||||||
args.add("10");
|
args.add("10");
|
||||||
args.add("-keySize");
|
args.add("-keySize");
|
||||||
args.add("10240");
|
args.add("10240");
|
||||||
Corona corona = new Corona(conf);
|
Freon freon = new Freon(conf);
|
||||||
int res = ToolRunner.run(conf, corona,
|
int res = ToolRunner.run(conf, freon,
|
||||||
args.toArray(new String[0]));
|
args.toArray(new String[0]));
|
||||||
Assert.assertEquals(10, corona.getNumberOfVolumesCreated());
|
Assert.assertEquals(10, freon.getNumberOfVolumesCreated());
|
||||||
Assert.assertEquals(10, corona.getNumberOfBucketsCreated());
|
Assert.assertEquals(10, freon.getNumberOfBucketsCreated());
|
||||||
Assert.assertEquals(100, corona.getNumberOfKeysAdded());
|
Assert.assertEquals(100, freon.getNumberOfKeysAdded());
|
||||||
Assert.assertEquals(0, res);
|
Assert.assertEquals(0, res);
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -18,7 +18,7 @@
|
||||||
|
|
||||||
package org.apache.hadoop.test;
|
package org.apache.hadoop.test;
|
||||||
|
|
||||||
import org.apache.hadoop.ozone.tools.Corona;
|
import org.apache.hadoop.ozone.tools.Freon;
|
||||||
import org.apache.hadoop.util.ProgramDriver;
|
import org.apache.hadoop.util.ProgramDriver;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -35,7 +35,7 @@ public class OzoneTestDriver {
|
||||||
public OzoneTestDriver(ProgramDriver pgd) {
|
public OzoneTestDriver(ProgramDriver pgd) {
|
||||||
this.pgd = pgd;
|
this.pgd = pgd;
|
||||||
try {
|
try {
|
||||||
pgd.addClass("corona", Corona.class,
|
pgd.addClass("freon", Freon.class,
|
||||||
"Populates ozone with data.");
|
"Populates ozone with data.");
|
||||||
} catch(Throwable e) {
|
} catch(Throwable e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
|
|
Loading…
Reference in New Issue