Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDDS-1380. Add functonality to write from multiple clients in MiniOzoneChaosCluster. Contributed by Shashikant Banerjee.

This commit is contained in:
Shashikant Banerjee 2019-04-16 18:52:07 +05:30
parent 008766c119
commit bfcb6534cd
2 changed files with 19 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java
View File

@ -56,10 +56,11 @@ public class MiniOzoneLoadGenerator {
private AtomicBoolean isWriteThreadRunning; private AtomicBoolean isWriteThreadRunning;
private final OzoneBucket ozoneBucket; private final List<OzoneBucket> ozoneBuckets;
MiniOzoneLoadGenerator(OzoneBucket bucket, int numThreads, int numBuffers) { MiniOzoneLoadGenerator(List<OzoneBucket> bucket, int numThreads,
this.ozoneBucket = bucket; int numBuffers) {
this.ozoneBuckets = bucket;
this.numWriteThreads = numThreads; this.numWriteThreads = numThreads;
this.numBuffers = numBuffers; this.numBuffers = numBuffers;
this.writeExecutor = new ThreadPoolExecutor(numThreads, numThreads, 100, this.writeExecutor = new ThreadPoolExecutor(numThreads, numThreads, 100,
@ -94,7 +95,9 @@ public class MiniOzoneLoadGenerator {
int bufferCapacity = buffer.capacity(); int bufferCapacity = buffer.capacity();
String keyName = threadName + "-" + index; String keyName = threadName + "-" + index;
try (OzoneOutputStream stream = ozoneBucket.createKey(keyName, OzoneBucket bucket =
ozoneBuckets.get((int) (Math.random() * ozoneBuckets.size()));
try (OzoneOutputStream stream = bucket.createKey(keyName,
bufferCapacity, ReplicationType.RATIS, ReplicationFactor.THREE, bufferCapacity, ReplicationType.RATIS, ReplicationFactor.THREE,
new HashMap<>())) { new HashMap<>())) {
stream.write(buffer.array()); stream.write(buffer.array());
@ -106,7 +109,7 @@ public class MiniOzoneLoadGenerator {
// to closed container. add a break here once that is fixed. // to closed container. add a break here once that is fixed.
} }
try (OzoneInputStream stream = ozoneBucket.readKey(keyName)) { try (OzoneInputStream stream = bucket.readKey(keyName)) {
byte[] readBuffer = new byte[bufferCapacity]; byte[] readBuffer = new byte[bufferCapacity];
int readLen = stream.read(readBuffer); int readLen = stream.read(readBuffer);

14
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
View File

@ -29,7 +29,8 @@ import picocli.CommandLine.Command;
import picocli.CommandLine.Option; import picocli.CommandLine.Option;
import picocli.CommandLine; import picocli.CommandLine;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
/** /**
@ -55,6 +56,10 @@ public class TestMiniChaosOzoneCluster implements Runnable {
description = "total run time") description = "total run time")
private static int numMinutes = 1440; // 1 day by default private static int numMinutes = 1440; // 1 day by default
@Option(names = {"-n", "--numClients"},
description = "no of clients writing to OM")
private static int numClients = 3;
@Option(names = {"-i", "--failureInterval"}, @Option(names = {"-i", "--failureInterval"},
description = "time between failure events in seconds") description = "time between failure events in seconds")
private static int failureInterval = 5; // 5 second period between failures. private static int failureInterval = 5; // 5 second period between failures.
@ -74,9 +79,12 @@ public class TestMiniChaosOzoneCluster implements Runnable {
store.createVolume(volumeName); store.createVolume(volumeName);
OzoneVolume volume = store.getVolume(volumeName); OzoneVolume volume = store.getVolume(volumeName);
volume.createBucket(bucketName); volume.createBucket(bucketName);
OzoneBucket ozoneBucket = volume.getBucket(bucketName); List<OzoneBucket> ozoneBuckets = new ArrayList<>(numClients);
for (int i = 0; i < numClients; i++) {
ozoneBuckets.add(volume.getBucket(bucketName));
}
loadGenerator = loadGenerator =
new MiniOzoneLoadGenerator(ozoneBucket, numThreads, numBuffers); new MiniOzoneLoadGenerator(ozoneBuckets, numThreads, numBuffers);
} }
/** /**