HDFS-11693. Ozone:Add archive support to containers. Contributed by Anu Engineer.
This commit is contained in:
parent
162dac96ce
commit
9c9be9f7f7
|
@ -74,25 +74,27 @@ import "Ozone.proto";
|
|||
* 16. GetSmallFile - A single RPC that combines both getKey and ReadChunk.
|
||||
*
|
||||
* 17. CloseContainer - Closes an open container and makes it immutable.
|
||||
*
|
||||
* 18. CopyContainer - Copies a container from a remote machine.
|
||||
*/
|
||||
|
||||
enum Type {
|
||||
CreateContainer = 1;
|
||||
ReadContainer = 2;
|
||||
UpdateContainer = 3;
|
||||
DeleteContainer = 4;
|
||||
ListContainer = 5;
|
||||
CreateContainer = 1;
|
||||
ReadContainer = 2;
|
||||
UpdateContainer = 3;
|
||||
DeleteContainer = 4;
|
||||
ListContainer = 5;
|
||||
|
||||
PutKey = 6;
|
||||
GetKey = 7;
|
||||
DeleteKey = 8;
|
||||
ListKey = 9;
|
||||
PutKey = 6;
|
||||
GetKey = 7;
|
||||
DeleteKey = 8;
|
||||
ListKey = 9;
|
||||
|
||||
ReadChunk = 10;
|
||||
DeleteChunk = 11;
|
||||
WriteChunk = 12;
|
||||
ListChunk = 13;
|
||||
CompactChunk = 14;
|
||||
ReadChunk = 10;
|
||||
DeleteChunk = 11;
|
||||
WriteChunk = 12;
|
||||
ListChunk = 13;
|
||||
CompactChunk = 14;
|
||||
|
||||
/** Combines Key and Chunk Operation into Single RPC. */
|
||||
PutSmallFile = 15;
|
||||
|
@ -128,6 +130,8 @@ enum Result {
|
|||
ERROR_CONTAINER_NOT_EMPTY = 23;
|
||||
ERROR_IN_COMPACT_DB = 24;
|
||||
UNCLOSED_CONTAINER_IO = 25;
|
||||
DELETE_ON_OPEN_CONTAINER = 26;
|
||||
CLOSED_CONTAINER_RETRY = 27;
|
||||
}
|
||||
|
||||
message ContainerCommandRequestProto {
|
||||
|
@ -382,3 +386,18 @@ message GetSmallFileRequestProto {
|
|||
message GetSmallFileResponseProto {
|
||||
required ReadChunkResponseProto data = 1;
|
||||
}
|
||||
|
||||
message CopyContainerRequestProto {
|
||||
required string containerName = 1;
|
||||
required uint64 readOffset = 2;
|
||||
optional uint64 len = 3;
|
||||
}
|
||||
|
||||
message CopyContainerResponseProto {
|
||||
required string archiveName = 1;
|
||||
required uint64 readOffset = 2;
|
||||
required uint64 len = 3;
|
||||
required bool eof = 4;
|
||||
repeated bytes data = 5;
|
||||
optional int64 checksum = 6;
|
||||
}
|
||||
|
|
|
@ -0,0 +1,106 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.scm;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang3.RandomStringUtils;
|
||||
import org.apache.hadoop.fs.FileUtil;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TemporaryFolder;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Iterator;
|
||||
import java.util.Random;
|
||||
import java.util.zip.Adler32;
|
||||
import java.util.zip.Checksum;
|
||||
|
||||
/**
|
||||
* Test archive creation and unpacking.
|
||||
*/
|
||||
public class TestArchive {
|
||||
private static final int DIR_COUNT = 10;
|
||||
private static final int SUB_DIR_COUNT = 3;
|
||||
private static final int FILE_COUNT = 10;
|
||||
@Rule
|
||||
public TemporaryFolder folder = new TemporaryFolder();
|
||||
|
||||
@Rule
|
||||
public TemporaryFolder outputFolder = new TemporaryFolder();
|
||||
|
||||
Checksum crc = new Adler32();
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
Random r = new Random();
|
||||
final int megaByte = 1024 * 1024;
|
||||
|
||||
for (int x = 0; x < DIR_COUNT; x++) {
|
||||
File subdir = folder.newFolder(String.format("dir%d", x));
|
||||
for (int y = 0; y < SUB_DIR_COUNT; y++) {
|
||||
File targetDir = new File(subdir.getPath().concat(File.separator)
|
||||
.concat(String.format("subdir%d%d", x, y)));
|
||||
if(!targetDir.mkdirs()) {
|
||||
throw new IOException("Failed to create subdirectory. " +
|
||||
targetDir.toString());
|
||||
}
|
||||
for (int z = 0; z < FILE_COUNT; z++) {
|
||||
Path temp = Paths.get(targetDir.getPath().concat(File.separator)
|
||||
.concat(String.format("File%d.txt", z)));
|
||||
byte[] buf = RandomStringUtils.randomAlphanumeric(r.nextInt(megaByte))
|
||||
.getBytes("UTF-8");
|
||||
Files.write(temp, buf);
|
||||
crc.update(buf, 0, buf.length);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testArchive() throws Exception {
|
||||
Checksum readCrc = new Adler32();
|
||||
File archiveFile = new File(outputFolder.getRoot() + File.separator
|
||||
+ "test.container.zip");
|
||||
long zipCheckSum = FileUtil.zip(folder.getRoot(), archiveFile);
|
||||
Assert.assertTrue(zipCheckSum > 0);
|
||||
File decomp = new File(outputFolder.getRoot() + File.separator +
|
||||
"decompress");
|
||||
if (!decomp.exists() && !decomp.mkdirs()) {
|
||||
throw new IOException("Unable to create the destination directory. " +
|
||||
decomp.getPath());
|
||||
}
|
||||
|
||||
FileUtil.unZip(archiveFile, decomp);
|
||||
String[] patterns = {"txt"};
|
||||
Iterator<File> iter = FileUtils.iterateFiles(decomp, patterns, true);
|
||||
int count = 0;
|
||||
while (iter.hasNext()) {
|
||||
count++;
|
||||
byte[] buf = Files.readAllBytes(iter.next().toPath());
|
||||
readCrc.update(buf, 0, buf.length);
|
||||
}
|
||||
Assert.assertEquals(DIR_COUNT * SUB_DIR_COUNT * FILE_COUNT, count);
|
||||
Assert.assertEquals(crc.getValue(), readCrc.getValue());
|
||||
}
|
||||
}
|
|
@ -0,0 +1,21 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.scm;
|
||||
/**
|
||||
Test cases for SCM client classes.
|
||||
*/
|
Loading…
Reference in New Issue