Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-13149. Ozone: Rename Corona to Freon. Contributed by Anu Engineer.

This commit is contained in:
Anu Engineer 2018-02-15 13:50:48 -08:00 committed by Owen O'Malley
parent cd807c9890
commit 136b419594
6 changed files with 96 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
View File

@ -46,6 +46,7 @@ function hadoop_usage
hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables"
hadoop_add_subcommand "ec" admin "run a HDFS ErasureCoding CLI"
hadoop_add_subcommand "fetchdt" client "fetch a delegation token from the NameNode"
hadoop_add_subcommand "freon" client "runs an ozone data generator"
hadoop_add_subcommand "fsck" admin "run a DFS filesystem checking utility"
hadoop_add_subcommand "getconf" client "get config values from configuration"
hadoop_add_subcommand "groups" client "get the groups which users belong to"
@ -139,6 +140,9 @@ function hdfscmd_case
fetchdt)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
;;
freon)
HADOOP_CLASSNAME=org.apache.hadoop.ozone.tools.Freon
;;
fsck)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSck
;;

114
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/Corona.java → hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/Freon.java
View File

@ -62,7 +62,7 @@
import static java.lang.Math.min;
/**
* Corona - A tool to populate ozone with data for testing.<br>
* Freon - A tool to populate ozone with data for testing.<br>
* This is not a map-reduce program and this is not for benchmarking
* Ozone write throughput.<br>
* It supports both online and offline modes. Default mode is offline,
@ -92,9 +92,9 @@
* can be used to override</li>
* </ul>
*/
public final class Corona extends Configured implements Tool {
public final class Freon extends Configured implements Tool {
enum CoronaOps {
enum FreonOps {
VOLUME_CREATE,
BUCKET_CREATE,
KEY_CREATE,
@ -127,7 +127,7 @@ enum CoronaOps {
private static final int QUANTILES = 10;
private static final Logger LOG =
LoggerFactory.getLogger(Corona.class);
LoggerFactory.getLogger(Freon.class);
private boolean printUsage = false;
private boolean completed = false;
@ -176,7 +176,7 @@ enum CoronaOps {
private ArrayList<Histogram> histograms = new ArrayList<>();
@VisibleForTesting
Corona(Configuration conf) throws IOException {
Freon(Configuration conf) throws IOException {
startTime = System.nanoTime();
jobStartTime = System.currentTimeMillis();
volumeCreationTime = new AtomicLong();
@ -189,7 +189,7 @@ enum CoronaOps {
numberOfKeysAdded = new AtomicLong();
ozoneClient = OzoneClientFactory.getClient(conf);
objectStore = ozoneClient.getObjectStore();
for (CoronaOps ops : CoronaOps.values()) {
for (FreonOps ops : FreonOps.values()) {
histograms.add(ops.ordinal(), new Histogram(new UniformReservoir()));
}
}
@ -199,7 +199,7 @@ enum CoronaOps {
*/
public static void main(String[] args) throws Exception {
Configuration conf = new OzoneConfiguration();
int res = ToolRunner.run(conf, new Corona(conf), args);
int res = ToolRunner.run(conf, new Freon(conf), args);
System.exit(res);
}
@ -271,7 +271,7 @@ private Options getOptions() {
OptionBuilder.withArgName("online | offline");
OptionBuilder.hasArg();
OptionBuilder.withDescription("specifies the mode of " +
"Corona run.");
"Freon run.");
Option optMode = OptionBuilder.create(MODE);
OptionBuilder.withArgName("source url");
@ -400,7 +400,7 @@ private void usage() {
System.out.println("-jsonDir "
+ "directory where json is created.");
System.out.println("-mode [online | offline] "
+ "specifies the mode in which Corona should run.");
+ "specifies the mode in which Freon should run.");
System.out.println("-source <url> "
+ "specifies the URL of s3 commoncrawl warc file to " +
"be used when the mode is online.");
@ -446,7 +446,7 @@ private Thread getProgressBarThread() {
}
/**
* Prints stats of {@link Corona} run to the PrintStream.
* Prints stats of {@link Freon} run to the PrintStream.
*
* @param out PrintStream
*/
@ -512,10 +512,10 @@ private void printStats(PrintStream out) {
if (jsonDir != null) {
String[][] quantileTime =
new String[CoronaOps.values().length][QUANTILES + 1];
String[] deviations = new String[CoronaOps.values().length];
String[] means = new String[CoronaOps.values().length];
for (CoronaOps ops : CoronaOps.values()) {
new String[FreonOps.values().length][QUANTILES + 1];
String[] deviations = new String[FreonOps.values().length];
String[] means = new String[FreonOps.values().length];
for (FreonOps ops : FreonOps.values()) {
Snapshot snapshot = histograms.get(ops.ordinal()).getSnapshot();
for (int i = 0; i <= QUANTILES; i++) {
quantileTime[ops.ordinal()][i] = DurationFormatUtils.formatDuration(
@ -531,26 +531,26 @@ private void printStats(PrintStream out) {
DURATION_FORMAT);
}
CoronaJobInfo jobInfo = new CoronaJobInfo().setExecTime(execTime)
FreonJobInfo jobInfo = new FreonJobInfo().setExecTime(execTime)
.setGitBaseRevision(VersionInfo.getRevision())
.setMeanVolumeCreateTime(means[CoronaOps.VOLUME_CREATE.ordinal()])
.setMeanVolumeCreateTime(means[FreonOps.VOLUME_CREATE.ordinal()])
.setDeviationVolumeCreateTime(
deviations[CoronaOps.VOLUME_CREATE.ordinal()])
deviations[FreonOps.VOLUME_CREATE.ordinal()])
.setTenQuantileVolumeCreateTime(
quantileTime[CoronaOps.VOLUME_CREATE.ordinal()])
.setMeanBucketCreateTime(means[CoronaOps.BUCKET_CREATE.ordinal()])
quantileTime[FreonOps.VOLUME_CREATE.ordinal()])
.setMeanBucketCreateTime(means[FreonOps.BUCKET_CREATE.ordinal()])
.setDeviationBucketCreateTime(
deviations[CoronaOps.BUCKET_CREATE.ordinal()])
deviations[FreonOps.BUCKET_CREATE.ordinal()])
.setTenQuantileBucketCreateTime(
quantileTime[CoronaOps.BUCKET_CREATE.ordinal()])
.setMeanKeyCreateTime(means[CoronaOps.KEY_CREATE.ordinal()])
.setDeviationKeyCreateTime(deviations[CoronaOps.KEY_CREATE.ordinal()])
quantileTime[FreonOps.BUCKET_CREATE.ordinal()])
.setMeanKeyCreateTime(means[FreonOps.KEY_CREATE.ordinal()])
.setDeviationKeyCreateTime(deviations[FreonOps.KEY_CREATE.ordinal()])
.setTenQuantileKeyCreateTime(
quantileTime[CoronaOps.KEY_CREATE.ordinal()])
.setMeanKeyWriteTime(means[CoronaOps.KEY_WRITE.ordinal()])
.setDeviationKeyWriteTime(deviations[CoronaOps.KEY_WRITE.ordinal()])
quantileTime[FreonOps.KEY_CREATE.ordinal()])
.setMeanKeyWriteTime(means[FreonOps.KEY_WRITE.ordinal()])
.setDeviationKeyWriteTime(deviations[FreonOps.KEY_WRITE.ordinal()])
.setTenQuantileKeyWriteTime(
quantileTime[CoronaOps.KEY_WRITE.ordinal()]);
quantileTime[FreonOps.KEY_WRITE.ordinal()]);
String jsonName =
new SimpleDateFormat("yyyyMMddHHmmss").format(Time.now()) + ".json";
String jsonPath = jsonDir + "/" + jsonName;
@ -704,7 +704,7 @@ public void run() {
objectStore.createVolume(volumeName);
long volumeCreationDuration = System.nanoTime() - start;
volumeCreationTime.getAndAdd(volumeCreationDuration);
histograms.get(CoronaOps.VOLUME_CREATE.ordinal())
histograms.get(FreonOps.VOLUME_CREATE.ordinal())
.update(volumeCreationDuration);
numberOfVolumesCreated.getAndIncrement();
volume = objectStore.getVolume(volumeName);
@ -724,7 +724,7 @@ public void run() {
start = System.nanoTime();
volume.createBucket(bucketName);
long bucketCreationDuration = System.nanoTime() - start;
histograms.get(CoronaOps.BUCKET_CREATE.ordinal())
histograms.get(FreonOps.BUCKET_CREATE.ordinal())
.update(bucketCreationDuration);
bucketCreationTime.getAndAdd(bucketCreationDuration);
numberOfBucketsCreated.getAndIncrement();
@ -741,7 +741,7 @@ public void run() {
OzoneOutputStream os =
bucket.createKey(key, keySize, type, factor);
long keyCreationDuration = System.nanoTime() - keyCreateStart;
histograms.get(CoronaOps.KEY_CREATE.ordinal())
histograms.get(FreonOps.KEY_CREATE.ordinal())
.update(keyCreationDuration);
keyCreationTime.getAndAdd(keyCreationDuration);
long keyWriteStart = System.nanoTime();
@ -750,7 +750,7 @@ public void run() {
os.close();
long keyWriteDuration = System.nanoTime() - keyWriteStart;
threadKeyWriteTime += keyWriteDuration;
histograms.get(CoronaOps.KEY_WRITE.ordinal())
histograms.get(FreonOps.KEY_WRITE.ordinal())
.update(keyWriteDuration);
totalBytesWritten.getAndAdd(keySize);
numberOfKeysAdded.getAndIncrement();
@ -780,7 +780,7 @@ public void run() {
}
private final class CoronaJobInfo {
private final class FreonJobInfo {
private String status;
private String gitBaseRevision;
@ -815,17 +815,17 @@ private final class CoronaJobInfo {
private String deviationKeyWriteTime;
private String[] tenQuantileKeyWriteTime;
private CoronaJobInfo() {
private FreonJobInfo() {
this.status = exception ? "Failed" : "Success";
this.numOfVolumes = Corona.this.numOfVolumes;
this.numOfBuckets = Corona.this.numOfBuckets;
this.numOfKeys = Corona.this.numOfKeys;
this.numOfThreads = Corona.this.numOfThreads;
this.keySize = Corona.this.keySize;
this.mode = Corona.this.mode;
this.jobStartTime = Time.formatTime(Corona.this.jobStartTime);
this.replicationFactor = Corona.this.factor.name();
this.replicationType = Corona.this.type.name();
this.numOfVolumes = Freon.this.numOfVolumes;
this.numOfBuckets = Freon.this.numOfBuckets;
this.numOfKeys = Freon.this.numOfKeys;
this.numOfThreads = Freon.this.numOfThreads;
this.keySize = Freon.this.keySize;
this.mode = Freon.this.mode;
this.jobStartTime = Time.formatTime(Freon.this.jobStartTime);
this.replicationFactor = Freon.this.factor.name();
this.replicationType = Freon.this.type.name();
long totalBytes =
Long.parseLong(numOfVolumes) * Long.parseLong(numOfBuckets) * Long
@ -833,7 +833,7 @@ private CoronaJobInfo() {
this.dataWritten = getInStorageUnits((double) totalBytes);
this.totalThroughputPerSecond = getInStorageUnits(
(totalBytes * 1.0) / TimeUnit.NANOSECONDS
.toSeconds(Corona.this.keyWriteTime.get() / threadPoolSize));
.toSeconds(Freon.this.keyWriteTime.get() / threadPoolSize));
}
private String getInStorageUnits(Double value) {
@ -858,81 +858,81 @@ private String getInStorageUnits(Double value) {
return size + " " + unit;
}
public CoronaJobInfo setGitBaseRevision(String gitBaseRevisionVal) {
public FreonJobInfo setGitBaseRevision(String gitBaseRevisionVal) {
gitBaseRevision = gitBaseRevisionVal;
return this;
}
public CoronaJobInfo setExecTime(String execTimeVal) {
public FreonJobInfo setExecTime(String execTimeVal) {
execTime = execTimeVal;
return this;
}
public CoronaJobInfo setMeanKeyWriteTime(String deviationKeyWriteTimeVal) {
public FreonJobInfo setMeanKeyWriteTime(String deviationKeyWriteTimeVal) {
this.meanKeyWriteTime = deviationKeyWriteTimeVal;
return this;
}
public CoronaJobInfo setDeviationKeyWriteTime(
public FreonJobInfo setDeviationKeyWriteTime(
String deviationKeyWriteTimeVal) {
this.deviationKeyWriteTime = deviationKeyWriteTimeVal;
return this;
}
public CoronaJobInfo setTenQuantileKeyWriteTime(
public FreonJobInfo setTenQuantileKeyWriteTime(
String[] tenQuantileKeyWriteTimeVal) {
this.tenQuantileKeyWriteTime = tenQuantileKeyWriteTimeVal;
return this;
}
public CoronaJobInfo setMeanKeyCreateTime(String deviationKeyWriteTimeVal) {
public FreonJobInfo setMeanKeyCreateTime(String deviationKeyWriteTimeVal) {
this.meanKeyCreateTime = deviationKeyWriteTimeVal;
return this;
}
public CoronaJobInfo setDeviationKeyCreateTime(
public FreonJobInfo setDeviationKeyCreateTime(
String deviationKeyCreateTimeVal) {
this.deviationKeyCreateTime = deviationKeyCreateTimeVal;
return this;
}
public CoronaJobInfo setTenQuantileKeyCreateTime(
public FreonJobInfo setTenQuantileKeyCreateTime(
String[] tenQuantileKeyCreateTimeVal) {
this.tenQuantileKeyCreateTime = tenQuantileKeyCreateTimeVal;
return this;
}
public CoronaJobInfo setMeanBucketCreateTime(
public FreonJobInfo setMeanBucketCreateTime(
String deviationKeyWriteTimeVal) {
this.meanBucketCreateTime = deviationKeyWriteTimeVal;
return this;
}
public CoronaJobInfo setDeviationBucketCreateTime(
public FreonJobInfo setDeviationBucketCreateTime(
String deviationBucketCreateTimeVal) {
this.deviationBucketCreateTime = deviationBucketCreateTimeVal;
return this;
}
public CoronaJobInfo setTenQuantileBucketCreateTime(
public FreonJobInfo setTenQuantileBucketCreateTime(
String[] tenQuantileBucketCreateTimeVal) {
this.tenQuantileBucketCreateTime = tenQuantileBucketCreateTimeVal;
return this;
}
public CoronaJobInfo setMeanVolumeCreateTime(
public FreonJobInfo setMeanVolumeCreateTime(
String deviationKeyWriteTimeVal) {
this.meanVolumeCreateTime = deviationKeyWriteTimeVal;
return this;
}
public CoronaJobInfo setDeviationVolumeCreateTime(
public FreonJobInfo setDeviationVolumeCreateTime(
String deviationVolumeCreateTimeVal) {
this.deviationVolumeCreateTime = deviationVolumeCreateTimeVal;
return this;
}
public CoronaJobInfo setTenQuantileVolumeCreateTime(
public FreonJobInfo setTenQuantileVolumeCreateTime(
String[] tenQuantileVolumeCreateTimeVal) {
this.tenQuantileVolumeCreateTime = tenQuantileVolumeCreateTimeVal;
return this;

8
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm
View File

@ -61,18 +61,18 @@ the background.
- `docker-compose up -d`
Now let us run some work load against ozone, to do that we will run corona.
Now let us run some work load against ozone, to do that we will run freon.
This will log into the datanode and run bash.
- `docker-compose exec datanode bash`
- `cd hadoop/bin`
Now you can run the oz command shell or corona the ozone load generator.
Now you can run the oz command shell or freon, the ozone load generator.
This is the command to run corona.
This is the command to run freon.
- `./hdfs corona -mode offline -validateWrites -numOfVolumes 1 -numOfBuckets 10 -numOfKeys 100`
- `./hdfs freon -mode offline -validateWrites -numOfVolumes 1 -numOfBuckets 10 -numOfKeys 100`
You can checkout the KSM UI to see the requests information.

4
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneOverview.md
View File

@ -22,11 +22,11 @@ keys. 
The following is a high-level overview of the core components of Ozone.
The main elements of Ozone are
:
### Clients
Ozone ships with a set of ready-made clients. They are 
Ozone CLI and Corona.

Ozone ships with a set of ready-made clients. They are 
Ozone CLI and Freon.

* [Ozone CLI](./OzoneCommandShell.html) is the command line interface like 'hdfs' command.

* Corona is a load generation tool for Ozone.

* Freon is a load generation tool for Ozone.

### REST Handler
Ozone provides both an RPC (Remote Procedure Call) as well as a REST

54
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/tools/TestCorona.java → hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/tools/TestFreon.java
View File

@ -36,9 +36,9 @@
import java.util.List;
/**
* Tests Corona, with MiniOzoneCluster.
* Tests Freon, with MiniOzoneCluster.
*/
public class TestCorona {
public class TestFreon {
private static MiniOzoneCluster cluster;
private static OzoneConfiguration conf;
@ -80,13 +80,13 @@ public void defaultTest() throws Exception {
args.add("5");
args.add("-numOfKeys");
args.add("10");
Corona corona = new Corona(conf);
int res = ToolRunner.run(conf, corona,
Freon freon = new Freon(conf);
int res = ToolRunner.run(conf, freon,
args.toArray(new String[0]));
Assert.assertEquals(2, corona.getNumberOfVolumesCreated());
Assert.assertEquals(10, corona.getNumberOfBucketsCreated());
Assert.assertEquals(100, corona.getNumberOfKeysAdded());
Assert.assertEquals(10240 - 36, corona.getKeyValueLength());
Assert.assertEquals(2, freon.getNumberOfVolumesCreated());
Assert.assertEquals(10, freon.getNumberOfBucketsCreated());
Assert.assertEquals(100, freon.getNumberOfKeysAdded());
Assert.assertEquals(10240 - 36, freon.getKeyValueLength());
Assert.assertEquals(0, res);
}
@ -103,17 +103,17 @@ public void validateWriteTest() throws Exception {
args.add("5");
args.add("-numOfKeys");
args.add("10");
Corona corona = new Corona(conf);
int res = ToolRunner.run(conf, corona,
Freon freon = new Freon(conf);
int res = ToolRunner.run(conf, freon,
args.toArray(new String[0]));
Assert.assertEquals(0, res);
Assert.assertEquals(2, corona.getNumberOfVolumesCreated());
Assert.assertEquals(10, corona.getNumberOfBucketsCreated());
Assert.assertEquals(100, corona.getNumberOfKeysAdded());
Assert.assertTrue(corona.getValidateWrites());
Assert.assertNotEquals(0, corona.getTotalKeysValidated());
Assert.assertNotEquals(0, corona.getSuccessfulValidationCount());
Assert.assertEquals(0, corona.getUnsuccessfulValidationCount());
Assert.assertEquals(2, freon.getNumberOfVolumesCreated());
Assert.assertEquals(10, freon.getNumberOfBucketsCreated());
Assert.assertEquals(100, freon.getNumberOfKeysAdded());
Assert.assertTrue(freon.getValidateWrites());
Assert.assertNotEquals(0, freon.getTotalKeysValidated());
Assert.assertNotEquals(0, freon.getSuccessfulValidationCount());
Assert.assertEquals(0, freon.getUnsuccessfulValidationCount());
System.setOut(originalStream);
}
@ -130,12 +130,12 @@ public void multiThread() throws Exception {
args.add("10");
args.add("-keySize");
args.add("10240");
Corona corona = new Corona(conf);
int res = ToolRunner.run(conf, corona,
Freon freon = new Freon(conf);
int res = ToolRunner.run(conf, freon,
args.toArray(new String[0]));
Assert.assertEquals(10, corona.getNumberOfVolumesCreated());
Assert.assertEquals(10, corona.getNumberOfBucketsCreated());
Assert.assertEquals(100, corona.getNumberOfKeysAdded());
Assert.assertEquals(10, freon.getNumberOfVolumesCreated());
Assert.assertEquals(10, freon.getNumberOfBucketsCreated());
Assert.assertEquals(100, freon.getNumberOfKeysAdded());
Assert.assertEquals(0, res);
}
@ -154,12 +154,12 @@ public void ratisTest3() throws Exception {
args.add("10");
args.add("-keySize");
args.add("10240");
Corona corona = new Corona(conf);
int res = ToolRunner.run(conf, corona,
Freon freon = new Freon(conf);
int res = ToolRunner.run(conf, freon,
args.toArray(new String[0]));
Assert.assertEquals(10, corona.getNumberOfVolumesCreated());
Assert.assertEquals(10, corona.getNumberOfBucketsCreated());
Assert.assertEquals(100, corona.getNumberOfKeysAdded());
Assert.assertEquals(10, freon.getNumberOfVolumesCreated());
Assert.assertEquals(10, freon.getNumberOfBucketsCreated());
Assert.assertEquals(100, freon.getNumberOfKeysAdded());
Assert.assertEquals(0, res);
}
}

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/test/OzoneTestDriver.java
View File

@ -18,7 +18,7 @@
package org.apache.hadoop.test;
import org.apache.hadoop.ozone.tools.Corona;
import org.apache.hadoop.ozone.tools.Freon;
import org.apache.hadoop.util.ProgramDriver;
/**
@ -35,7 +35,7 @@ public OzoneTestDriver() {
public OzoneTestDriver(ProgramDriver pgd) {
this.pgd = pgd;
try {
pgd.addClass("corona", Corona.class,
pgd.addClass("freon", Freon.class,
"Populates ozone with data.");
} catch(Throwable e) {
e.printStackTrace();