HDFS-12591. [READ] Implement LevelDBFileRegionFormat. Contributed by Ewan Higgs.
This commit is contained in:
parent
352f994b64
commit
b634053c4d
|
@ -352,6 +352,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
|
||||||
public static final String DFS_PROVIDED_ALIASMAP_TEXT_CODEC = "dfs.provided.aliasmap.text.codec";
|
public static final String DFS_PROVIDED_ALIASMAP_TEXT_CODEC = "dfs.provided.aliasmap.text.codec";
|
||||||
public static final String DFS_PROVIDED_ALIASMAP_TEXT_WRITE_PATH = "dfs.provided.aliasmap.text.write.path";
|
public static final String DFS_PROVIDED_ALIASMAP_TEXT_WRITE_PATH = "dfs.provided.aliasmap.text.write.path";
|
||||||
|
|
||||||
|
public static final String DFS_PROVIDED_ALIASMAP_LEVELDB_PATH = "dfs.provided.aliasmap.leveldb.read.path";
|
||||||
|
|
||||||
public static final String DFS_LIST_LIMIT = "dfs.ls.limit";
|
public static final String DFS_LIST_LIMIT = "dfs.ls.limit";
|
||||||
public static final int DFS_LIST_LIMIT_DEFAULT = 1000;
|
public static final int DFS_LIST_LIMIT_DEFAULT = 1000;
|
||||||
public static final String DFS_CONTENT_SUMMARY_LIMIT_KEY = "dfs.content-summary.limit";
|
public static final String DFS_CONTENT_SUMMARY_LIMIT_KEY = "dfs.content-summary.limit";
|
||||||
|
|
|
@ -0,0 +1,257 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.hdfs.server.common.blockaliasmap.impl;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Optional;
|
||||||
|
|
||||||
|
import org.iq80.leveldb.DB;
|
||||||
|
import org.iq80.leveldb.DBIterator;
|
||||||
|
import static org.fusesource.leveldbjni.JniDBFactory.factory;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configurable;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.Block;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation;
|
||||||
|
import org.apache.hadoop.hdfs.server.common.FileRegion;
|
||||||
|
import org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap;
|
||||||
|
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PROVIDED_ALIASMAP_LEVELDB_PATH;
|
||||||
|
import static org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap.fromBlockBytes;
|
||||||
|
import static org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap.fromProvidedStorageLocationBytes;
|
||||||
|
import static org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap.toProtoBufBytes;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A LevelDB based implementation of {@link BlockAliasMap}.
|
||||||
|
*/
|
||||||
|
public class LevelDBFileRegionAliasMap
|
||||||
|
extends BlockAliasMap<FileRegion> implements Configurable {
|
||||||
|
|
||||||
|
private Configuration conf;
|
||||||
|
private LevelDBOptions opts = new LevelDBOptions();
|
||||||
|
|
||||||
|
public static final Logger LOG =
|
||||||
|
LoggerFactory.getLogger(LevelDBFileRegionAliasMap.class);
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setConf(Configuration conf) {
|
||||||
|
opts.setConf(conf);
|
||||||
|
this.conf = conf;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Configuration getConf() {
|
||||||
|
return conf;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Reader<FileRegion> getReader(Reader.Options opts) throws IOException {
|
||||||
|
if (null == opts) {
|
||||||
|
opts = this.opts;
|
||||||
|
}
|
||||||
|
if (!(opts instanceof LevelDBOptions)) {
|
||||||
|
throw new IllegalArgumentException("Invalid options " + opts.getClass());
|
||||||
|
}
|
||||||
|
LevelDBOptions o = (LevelDBOptions) opts;
|
||||||
|
return new LevelDBFileRegionAliasMap.LevelDBReader(
|
||||||
|
createDB(o.levelDBPath, false));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Writer<FileRegion> getWriter(Writer.Options opts) throws IOException {
|
||||||
|
if (null == opts) {
|
||||||
|
opts = this.opts;
|
||||||
|
}
|
||||||
|
if (!(opts instanceof LevelDBOptions)) {
|
||||||
|
throw new IllegalArgumentException("Invalid options " + opts.getClass());
|
||||||
|
}
|
||||||
|
LevelDBOptions o = (LevelDBOptions) opts;
|
||||||
|
return new LevelDBFileRegionAliasMap.LevelDBWriter(
|
||||||
|
createDB(o.levelDBPath, true));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static DB createDB(String levelDBPath, boolean createIfMissing)
|
||||||
|
throws IOException {
|
||||||
|
if (levelDBPath == null || levelDBPath.length() == 0) {
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"A valid path needs to be specified for "
|
||||||
|
+ LevelDBFileRegionAliasMap.class + " using the parameter "
|
||||||
|
+ DFS_PROVIDED_ALIASMAP_LEVELDB_PATH);
|
||||||
|
}
|
||||||
|
org.iq80.leveldb.Options options = new org.iq80.leveldb.Options();
|
||||||
|
options.createIfMissing(createIfMissing);
|
||||||
|
return factory.open(new File(levelDBPath), options);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void refresh() throws IOException {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
// Do nothing.
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Class specifying reader options for the {@link LevelDBFileRegionAliasMap}.
|
||||||
|
*/
|
||||||
|
public static class LevelDBOptions implements LevelDBReader.Options,
|
||||||
|
LevelDBWriter.Options, Configurable {
|
||||||
|
private Configuration conf;
|
||||||
|
private String levelDBPath;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setConf(Configuration conf) {
|
||||||
|
this.conf = conf;
|
||||||
|
this.levelDBPath = conf.get(DFS_PROVIDED_ALIASMAP_LEVELDB_PATH);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Configuration getConf() {
|
||||||
|
return conf;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public LevelDBOptions filename(String levelDBPath) {
|
||||||
|
this.levelDBPath = levelDBPath;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class is used as a reader for block maps which
|
||||||
|
* are stored as LevelDB files.
|
||||||
|
*/
|
||||||
|
public static class LevelDBReader extends Reader<FileRegion> {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Options for {@link LevelDBReader}.
|
||||||
|
*/
|
||||||
|
public interface Options extends Reader.Options {
|
||||||
|
Options filename(String levelDBPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
private DB db;
|
||||||
|
|
||||||
|
LevelDBReader(DB db) {
|
||||||
|
this.db = db;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Optional<FileRegion> resolve(Block block) throws IOException {
|
||||||
|
if (db == null) {
|
||||||
|
return Optional.empty();
|
||||||
|
}
|
||||||
|
// consider layering index w/ composable format
|
||||||
|
byte[] key = toProtoBufBytes(block);
|
||||||
|
byte[] value = db.get(key);
|
||||||
|
ProvidedStorageLocation psl = fromProvidedStorageLocationBytes(value);
|
||||||
|
return Optional.of(new FileRegion(block, psl));
|
||||||
|
}
|
||||||
|
|
||||||
|
static class FRIterator implements Iterator<FileRegion> {
|
||||||
|
private final DBIterator internal;
|
||||||
|
|
||||||
|
FRIterator(DBIterator internal) {
|
||||||
|
this.internal = internal;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean hasNext() {
|
||||||
|
return internal.hasNext();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public FileRegion next() {
|
||||||
|
Map.Entry<byte[], byte[]> entry = internal.next();
|
||||||
|
if (entry == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Block block = fromBlockBytes(entry.getKey());
|
||||||
|
ProvidedStorageLocation psl =
|
||||||
|
fromProvidedStorageLocationBytes(entry.getValue());
|
||||||
|
return new FileRegion(block, psl);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void remove() {
|
||||||
|
throw new UnsupportedOperationException();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Iterator<FileRegion> iterator() {
|
||||||
|
if (db == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
DBIterator iterator = db.iterator();
|
||||||
|
iterator.seekToFirst();
|
||||||
|
return new FRIterator(iterator);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
if (db != null) {
|
||||||
|
db.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class is used as a writer for block maps which
|
||||||
|
* are stored as LevelDB files.
|
||||||
|
*/
|
||||||
|
public static class LevelDBWriter extends Writer<FileRegion> {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Interface for Writer options.
|
||||||
|
*/
|
||||||
|
public interface Options extends Writer.Options {
|
||||||
|
Options filename(String levelDBPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
private final DB db;
|
||||||
|
|
||||||
|
LevelDBWriter(DB db) {
|
||||||
|
this.db = db;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void store(FileRegion token) throws IOException {
|
||||||
|
byte[] key = toProtoBufBytes(token.getBlock());
|
||||||
|
byte[] value = toProtoBufBytes(token.getProvidedStorageLocation());
|
||||||
|
db.put(key, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
if (db != null) {
|
||||||
|
db.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,115 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hdfs.server.common.blockaliasmap.impl;
|
||||||
|
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.Block;
|
||||||
|
import org.apache.hadoop.hdfs.server.common.FileRegion;
|
||||||
|
import org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.util.Iterator;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertNotEquals;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tests for the {@link LevelDBFileRegionAliasMap}.
|
||||||
|
*/
|
||||||
|
public class TestLevelDBFileRegionAliasMap {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A basic test to verify that we can write data and read it back again.
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testReadBack() throws Exception {
|
||||||
|
File dbFile = Files.createTempDirectory("fileregionformat")
|
||||||
|
.toFile();
|
||||||
|
try {
|
||||||
|
LevelDBFileRegionAliasMap frf = new LevelDBFileRegionAliasMap();
|
||||||
|
LevelDBFileRegionAliasMap.LevelDBOptions opts =
|
||||||
|
new LevelDBFileRegionAliasMap.LevelDBOptions()
|
||||||
|
.filename(dbFile.getAbsolutePath());
|
||||||
|
BlockAliasMap.Writer<FileRegion> writer = frf.getWriter(opts);
|
||||||
|
|
||||||
|
FileRegion fr = new FileRegion(1, new Path("/file"), 1, 1, 1);
|
||||||
|
writer.store(fr);
|
||||||
|
writer.close();
|
||||||
|
|
||||||
|
BlockAliasMap.Reader<FileRegion> reader = frf.getReader(opts);
|
||||||
|
FileRegion fr2 = reader.resolve(new Block(1, 1, 1)).get();
|
||||||
|
assertEquals(fr, fr2);
|
||||||
|
reader.close();
|
||||||
|
} finally {
|
||||||
|
dbFile.delete();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
/**
|
||||||
|
* A basic test to verify that we can read a bunch of data that we've written.
|
||||||
|
*/
|
||||||
|
public void testIterate() throws Exception {
|
||||||
|
FileRegion[] regions = new FileRegion[10];
|
||||||
|
regions[0] = new FileRegion(1, new Path("/file1"), 0, 1024, 1);
|
||||||
|
regions[1] = new FileRegion(2, new Path("/file1"), 1024, 1024, 1);
|
||||||
|
regions[2] = new FileRegion(3, new Path("/file1"), 2048, 1024, 1);
|
||||||
|
regions[3] = new FileRegion(4, new Path("/file2"), 0, 1024, 1);
|
||||||
|
regions[4] = new FileRegion(5, new Path("/file2"), 1024, 1024, 1);
|
||||||
|
regions[5] = new FileRegion(6, new Path("/file2"), 2048, 1024, 1);
|
||||||
|
regions[6] = new FileRegion(7, new Path("/file2"), 3072, 1024, 1);
|
||||||
|
regions[7] = new FileRegion(8, new Path("/file3"), 0, 1024, 1);
|
||||||
|
regions[8] = new FileRegion(9, new Path("/file4"), 0, 1024, 1);
|
||||||
|
regions[9] = new FileRegion(10, new Path("/file5"), 0, 1024, 1);
|
||||||
|
File dbFile = Files.createTempDirectory("fileregionformat")
|
||||||
|
.toFile();
|
||||||
|
try {
|
||||||
|
LevelDBFileRegionAliasMap frf = new LevelDBFileRegionAliasMap();
|
||||||
|
LevelDBFileRegionAliasMap.LevelDBOptions opts =
|
||||||
|
new LevelDBFileRegionAliasMap.LevelDBOptions()
|
||||||
|
.filename(dbFile.getAbsolutePath());
|
||||||
|
BlockAliasMap.Writer<FileRegion> writer = frf.getWriter(opts);
|
||||||
|
|
||||||
|
for (FileRegion fr : regions) {
|
||||||
|
writer.store(fr);
|
||||||
|
}
|
||||||
|
writer.close();
|
||||||
|
|
||||||
|
BlockAliasMap.Reader<FileRegion> reader = frf.getReader(opts);
|
||||||
|
Iterator<FileRegion> it = reader.iterator();
|
||||||
|
int last = -1;
|
||||||
|
int count = 0;
|
||||||
|
while(it.hasNext()) {
|
||||||
|
FileRegion fr = it.next();
|
||||||
|
int blockId = (int)fr.getBlock().getBlockId();
|
||||||
|
assertEquals(regions[blockId-1], fr);
|
||||||
|
assertNotEquals(blockId, last);
|
||||||
|
last = blockId;
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
assertEquals(count, 10);
|
||||||
|
|
||||||
|
reader.close();
|
||||||
|
} finally {
|
||||||
|
dbFile.delete();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue