Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-10195. Ozone: Add container persistence. Contributed by Anu Engineer.

This commit is contained in:
Chris Nauroth 2016-03-26 11:40:00 -07:00 committed by Owen O'Malley
parent c18c6801ef
commit e758f90f93
33 changed files with 1506 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
View File

@ -41,6 +41,9 @@ public final class OzoneConfigKeys {
"dfs.objectstore.trace.enabled";
public static final boolean DFS_OBJECTSTORE_TRACE_ENABLED_DEFAULT = false;
public static final String DFS_OZONE_METADATA_DIRS =
"dfs.ozone.metadata.dirs";
/**
* There is no need to instantiate this class.

16
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneConsts.java → hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hadoop.ozone.web.utils;
package org.apache.hadoop.ozone;
import org.apache.hadoop.classification.InterfaceAudience;
@ -53,6 +53,20 @@ public final class OzoneConsts {
public static final String OZONE_USER = "user";
public static final String OZONE_REQUEST = "request";
public static final String CONTAINER_EXTENSION = ".container";
public static final String CONTAINER_META = ".meta";
// container storage is in the following format.
// Data Volume basePath/containers/<containerName>/metadata and
// Data Volume basePath/containers/<containerName>/data/...
public static final String CONTAINER_PREFIX = "containers";
public static final String CONTAINER_META_PATH = "metadata";
public static final String CONTAINER_DATA_PATH = "data";
public static final String CONTAINER_ROOT_PREFIX = "repository";
public static final String CONTAINER_DB = "container.db";
/**
* Supports Bucket Versioning.

47
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
View File

@ -35,8 +35,9 @@ public class ContainerData {
private final String containerName;
private final Map<String, String> metadata;
private String path;
private String dbPath; // Path to Level DB Store.
// Path to Physical file system where container and checksum are stored.
private String containerFilePath;
/**
* Constructs a ContainerData Object.
@ -63,8 +64,13 @@ public class ContainerData {
}
if (protoData.hasContainerPath()) {
data.setPath(protoData.getContainerPath());
data.setContainerPath(protoData.getContainerPath());
}
if (protoData.hasDbPath()) {
data.setDBPath(protoData.getDbPath());
}
return data;
}
@ -77,9 +83,15 @@ public class ContainerData {
ContainerProtos.ContainerData.Builder builder = ContainerProtos
.ContainerData.newBuilder();
builder.setName(this.getContainerName());
if (this.getPath() != null) {
builder.setContainerPath(this.getPath());
if (this.getDBPath() != null) {
builder.setDbPath(this.getDBPath());
}
if (this.getContainerPath() != null) {
builder.setContainerPath(this.getContainerPath());
}
for (Map.Entry<String, String> entry : metadata.entrySet()) {
ContainerProtos.KeyValue.Builder keyValBuilder =
ContainerProtos.KeyValue.newBuilder();
@ -144,8 +156,8 @@ public class ContainerData {
*
* @return - path
*/
public String getPath() {
return path;
public String getDBPath() {
return dbPath;
}
/**
@ -153,8 +165,8 @@ public class ContainerData {
*
* @param path - String.
*/
public void setPath(String path) {
this.path = path;
public void setDBPath(String path) {
this.dbPath = path;
}
/**
@ -167,4 +179,21 @@ public class ContainerData {
public String getName() {
return getContainerName();
}
/**
* Get container file path.
* @return - Physical path where container file and checksum is stored.
*/
public String getContainerPath() {
return containerFilePath;
}
/**
* Set container Path.
* @param containerFilePath - File path.
*/
public void setContainerPath(String containerFilePath) {
this.containerFilePath = containerFilePath;
}
}

144
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
View File

@ -19,7 +19,23 @@
package org.apache.hadoop.ozone.container.common.helpers;
import com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.impl.ContainerManagerImpl;
import org.apache.hadoop.ozone.container.common.utils.LevelDBStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import static org.apache.commons.io.FilenameUtils.removeExtension;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_EXTENSION;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_META;
/**
* A set of helper functions to create proper responses.
@ -104,6 +120,134 @@ public final class ContainerUtils {
"Server does not support this command yet.").build();
}
/**
* get containerName from a container file.
*
* @param containerFile - File
* @return Name of the container.
*/
public static String getContainerNameFromFile(File containerFile) {
Preconditions.checkNotNull(containerFile);
return Paths.get(containerFile.getParent()).resolve(
removeExtension(containerFile.getName())).toString();
}
/**
* Verifies that this in indeed a new container.
*
* @param containerFile - Container File to verify
* @param metadataFile - metadata File to verify
* @throws IOException
*/
public static void verifyIsNewContainer(File containerFile, File metadataFile)
throws IOException {
Logger log = LoggerFactory.getLogger(ContainerManagerImpl.class);
if (containerFile.exists()) {
log.error("container already exists on disk. File: {}",
containerFile.toPath());
throw new FileAlreadyExistsException("container already exists on " +
"disk.");
}
if (metadataFile.exists()) {
log.error("metadata found on disk, but missing container. Refusing to" +
" write this container. File: {} ", metadataFile.toPath());
throw new FileAlreadyExistsException(("metadata found on disk, but " +
"missing container. Refusing to write this container."));
}
File parentPath = new File(containerFile.getParent());
if (!parentPath.exists() && !parentPath.mkdirs()) {
log.error("Unable to create parent path. Path: {}",
parentPath.toString());
throw new IOException("Unable to create container directory.");
}
if (!containerFile.createNewFile()) {
log.error("creation of a new container file failed. File: {}",
containerFile.toPath());
throw new IOException("creation of a new container file failed.");
}
if (!metadataFile.createNewFile()) {
log.error("creation of the metadata file failed. File: {}",
metadataFile.toPath());
throw new IOException("creation of a new container file failed.");
}
}
/**
* creates a Metadata DB for the specified container.
*
* @param containerPath - Container Path.
* @throws IOException
*/
public static Path createMetadata(Path containerPath) throws IOException {
Preconditions.checkNotNull(containerPath);
containerPath = containerPath.resolve(OzoneConsts.CONTAINER_META_PATH);
if (!containerPath.toFile().mkdirs()) {
throw new IOException("Unable to create directory for metadata storage." +
" Path {}" + containerPath);
}
containerPath = containerPath.resolve(OzoneConsts.CONTAINER_DB);
LevelDBStore store = new LevelDBStore(containerPath.toFile(), true);
// we close since the SCM pre-creates containers.
// we will open and put Db handle into a cache when keys are being created
// in a container.
store.close();
return containerPath;
}
/**
* remove Container if it is empty.
* <p/>
* There are three things we need to delete.
* <p/>
* 1. Container file and metadata file. 2. The Level DB file 3. The path that
* we created on the data location.
*
* @param containerData - Data of the container to remove.
* @throws IOException
*/
public static void removeContainer(ContainerData containerData) throws
IOException {
Preconditions.checkNotNull(containerData);
// TODO : Check if there are any keys. This needs to be done
// by calling into key layer code, hence this is a TODO for now.
Path dbPath = Paths.get(containerData.getDBPath());
// Delete the DB File.
FileUtils.forceDelete(dbPath.toFile());
dbPath = dbPath.getParent();
// Delete all Metadata in the Data directories for this containers.
if (dbPath != null) {
FileUtils.deleteDirectory(dbPath.toFile());
dbPath = dbPath.getParent();
}
// now delete the container directory, this means that all key data dirs
// will be removed too.
if (dbPath != null) {
FileUtils.deleteDirectory(dbPath.toFile());
}
// Delete the container metadata from the metadata locations.
String rootPath = getContainerNameFromFile(new File(containerData
.getContainerPath()));
Path containerPath = Paths.get(rootPath.concat(CONTAINER_EXTENSION));
Path metaPath = Paths.get(rootPath.concat(CONTAINER_META));
FileUtils.forceDelete(containerPath.toFile());
FileUtils.forceDelete(metaPath.toFile());
}
private ContainerUtils() {
//never constructed.
}

113
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java Normal file
View File

@ -0,0 +1,113 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.container.common.impl;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerLocationManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.LinkedList;
import java.util.List;
/**
* A class that tells the ContainerManager where to place the containers.
* Please note : There is *no* one-to-one correlation between metadata
* locations and data locations.
*
* For example : A user could map all container files to a
* SSD but leave data/metadata on bunch of other disks.
*/
public class ContainerLocationManagerImpl implements ContainerLocationManager {
private static final Logger LOG =
LoggerFactory.getLogger(ContainerLocationManagerImpl.class);
private final Configuration conf;
private final FsDatasetSpi dataset;
private final Path[] volumePaths;
private int currentIndex;
private final List<Path> locations;
/**
* Constructs a Location Manager.
* @param conf - Configuration.
*/
public ContainerLocationManagerImpl(Configuration conf, List<Path> locations,
FsDatasetSpi dataset) throws IOException {
this.conf = conf;
this.dataset = dataset;
List<Path> pathList = new LinkedList<>();
FsDatasetSpi.FsVolumeReferences references;
try {
synchronized (this.dataset) {
references = this.dataset.getFsVolumeReferences();
for (int ndx = 0; ndx < references.size(); ndx++) {
FsVolumeSpi vol = references.get(ndx);
pathList.add(Paths.get(vol.getBasePath()));
}
references.close();
volumePaths = pathList.toArray(new Path[pathList.size()]);
this.locations = locations;
}
} catch (IOException ex) {
LOG.error("Unable to get volume paths.", ex);
throw new IOException("Internal error", ex);
}
}
/**
* Returns the path where the container should be placed from a set of
* locations.
*
* @return A path where we should place this container and metadata.
* @throws IOException
*/
@Override
public Path getContainerPath()
throws IOException {
Preconditions.checkState(locations.size() > 0);
int index = currentIndex % locations.size();
return locations.get(index).resolve(OzoneConsts.CONTAINER_ROOT_PREFIX);
}
/**
* Returns the path where the container Data file are stored.
*
* @return a path where we place the LevelDB and data files of a container.
* @throws IOException
*/
@Override
public Path getDataPath(String containerName) throws IOException {
Path currentPath = volumePaths[currentIndex++ % volumePaths.length];
currentPath = currentPath.resolve(OzoneConsts.CONTAINER_PREFIX);
return currentPath.resolve(containerName);
}
}

514
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java Normal file
View File

@ -0,0 +1,514 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.container.common.impl;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.helpers.Pipeline;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerLocationManager;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.DigestInputStream;
import java.security.DigestOutputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_EXTENSION;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_META;
/**
* A Generic ContainerManagerImpl that will be called from Ozone
* ContainerManagerImpl. This allows us to support delta changes to ozone
* version without having to rewrite the containerManager.
*/
public class ContainerManagerImpl implements ContainerManager {
static final Logger LOG =
LoggerFactory.getLogger(ContainerManagerImpl.class);
private final ConcurrentSkipListMap<String, ContainerStatus>
containerMap = new ConcurrentSkipListMap<>();
// This lock follows fair locking policy of first-come first-serve
// for waiting threads.
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
private ContainerLocationManager locationManager;
/**
* Init call that sets up a container Manager.
*
* @param config - Configuration.
* @param containerDirs - List of Metadata Container locations.
* @throws IOException
*/
@Override
public void init(Configuration config, List<Path> containerDirs,
FsDatasetSpi dataset)
throws IOException {
Preconditions.checkNotNull(config);
Preconditions.checkNotNull(containerDirs);
Preconditions.checkState(containerDirs.size() > 0);
readLock();
try {
for (Path path : containerDirs) {
File directory = path.toFile();
if (!directory.isDirectory()) {
LOG.error("Invalid path to container metadata directory. path: {}",
path.toString());
throw new IOException("Invalid path to container metadata directory" +
". " + path);
}
File[] files = directory.listFiles(new ContainerFilter());
if (files != null) {
for (File containerFile : files) {
String containerPath =
ContainerUtils.getContainerNameFromFile(containerFile);
Preconditions.checkNotNull(containerPath);
readContainerInfo(containerPath);
}
}
}
this.locationManager = new ContainerLocationManagerImpl(config,
containerDirs, dataset);
} finally {
readUnlock();
}
}
/**
* Reads the Container Info from a file and verifies that checksum match. If
* the checksums match, then that file is added to containerMap.
*
* @param containerName - Name which points to the persisted container.
*/
private void readContainerInfo(String containerName)
throws IOException {
Preconditions.checkState(containerName.length() > 0);
FileInputStream containerStream = null;
DigestInputStream dis = null;
FileInputStream metaStream = null;
Path cPath = Paths.get(containerName).getFileName();
String keyName = null;
if (cPath != null) {
keyName = cPath.toString();
}
Preconditions.checkNotNull(keyName);
try {
String containerFileName = containerName.concat(CONTAINER_EXTENSION);
String metaFileName = containerName.concat(CONTAINER_META);
containerStream = new FileInputStream(containerFileName);
metaStream = new FileInputStream(metaFileName);
MessageDigest sha = MessageDigest.getInstance("SHA-256");
dis = new DigestInputStream(containerStream, sha);
ContainerData containerData = ContainerData.getFromProtBuf(
ContainerProtos.ContainerData.parseDelimitedFrom(dis));
ContainerProtos.ContainerMeta meta = ContainerProtos.ContainerMeta
.parseDelimitedFrom(metaStream);
if (meta != null && !DigestUtils.sha256Hex(sha.digest()).equals(meta
.getHash())) {
throw new IOException("Invalid SHA found for file.");
}
containerMap.put(keyName, new ContainerStatus(containerData, true));
} catch (IOException | NoSuchAlgorithmException ex) {
LOG.error("read failed for file: {} ex: {}",
containerName, ex.getMessage());
// TODO : Add this file to a recovery Queue.
// Remember that this container is busted and we cannot use it.
containerMap.put(keyName, new ContainerStatus(null, false));
} finally {
IOUtils.closeStream(dis);
IOUtils.closeStream(containerStream);
IOUtils.closeStream(metaStream);
}
}
/**
* Creates a container with the given name.
*
* @param pipeline -- Nodes which make up this container.
* @param containerData - Container Name and metadata.
* @throws IOException
*/
@Override
public void createContainer(Pipeline pipeline, ContainerData containerData)
throws IOException {
Preconditions.checkNotNull(containerData);
writeLock();
try {
if (containerMap.containsKey(containerData.getName())) {
throw new FileAlreadyExistsException("container already exists.");
}
// This is by design. We first write and close the
// container Info and metadata to a directory.
// Then read back and put that info into the containerMap.
// This allows us to make sure that our write is consistent.
writeContainerInfo(containerData);
File cFile = new File(containerData.getContainerPath());
readContainerInfo(ContainerUtils.getContainerNameFromFile(cFile));
} catch (NoSuchAlgorithmException ex) {
throw new IOException("failed to create container", ex);
} finally {
writeUnlock();
}
}
/**
* Writes a container to a chosen location and updates the container Map.
*
* The file formats of ContainerData and Container Meta is the following.
*
* message ContainerData {
* required string name = 1;
* repeated KeyValue metadata = 2;
* optional string dbPath = 3;
* optional string containerPath = 4;
* }
*
* message ContainerMeta {
* required string fileName = 1;
* required string hash = 2;
* }
*
* @param containerData - container Data
*/
private void writeContainerInfo(ContainerData containerData)
throws IOException, NoSuchAlgorithmException {
Preconditions.checkNotNull(this.locationManager);
FileOutputStream containerStream = null;
DigestOutputStream dos = null;
FileOutputStream metaStream = null;
Path location = locationManager.getContainerPath();
File containerFile = location.resolve(containerData
.getContainerName().concat(CONTAINER_EXTENSION))
.toFile();
File metadataFile = location.resolve(containerData
.getContainerName().concat(CONTAINER_META))
.toFile();
try {
ContainerUtils.verifyIsNewContainer(containerFile, metadataFile);
Path metadataPath = this.locationManager.getDataPath(
containerData.getContainerName());
metadataPath = ContainerUtils.createMetadata(metadataPath);
containerStream = new FileOutputStream(containerFile);
metaStream = new FileOutputStream(metadataFile);
MessageDigest sha = MessageDigest.getInstance("SHA-256");
dos = new DigestOutputStream(containerStream, sha);
containerData.setDBPath(metadataPath.toString());
containerData.setContainerPath(containerFile.toString());
ContainerProtos.ContainerData protoData = containerData
.getProtoBufMessage();
protoData.writeDelimitedTo(dos);
ContainerProtos.ContainerMeta protoMeta = ContainerProtos
.ContainerMeta.newBuilder()
.setFileName(containerFile.toString())
.setHash(DigestUtils.sha256Hex(sha.digest()))
.build();
protoMeta.writeDelimitedTo(metaStream);
} catch (IOException ex) {
// TODO : we need to clean up partially constructed files
// The proper way to do would be for a thread
// to read all these 3 artifacts and make sure they are
// sane. That info needs to come from the replication
// pipeline, and if not consistent delete these file.
// In case of ozone this is *not* a deal breaker since
// SCM is guaranteed to generate unique container names.
LOG.error("creation of container failed. Name: {} "
, containerData.getContainerName());
throw ex;
} finally {
IOUtils.closeStream(dos);
IOUtils.closeStream(containerStream);
IOUtils.closeStream(metaStream);
}
}
/**
* Deletes an existing container.
*
* @param pipeline - nodes that make this container.
* @param containerName - name of the container.
* @throws IOException
*/
@Override
public void deleteContainer(Pipeline pipeline, String containerName) throws
IOException {
Preconditions.checkState(containerName.length() > 0);
writeLock();
try {
ContainerStatus status = containerMap.get(containerName);
if (status == null) {
LOG.info("No such container. Name: {}", containerName);
throw new IOException("No such container. Name : " + containerName);
}
ContainerUtils.removeContainer(status.containerData);
containerMap.remove(containerName);
} finally {
writeUnlock();
}
}
/**
* A simple interface for container Iterations.
* <p/>
* This call make no guarantees about consistency of the data between
* different list calls. It just returns the best known data at that point of
* time. It is possible that using this iteration you can miss certain
* container from the listing.
*
* @param prevKey - Previous Key Value or empty String.
* @param count - how many to return
* @param data - Actual containerData
* @throws IOException
*/
@Override
public void listContainer(String prevKey, long count,
List<ContainerData> data) throws IOException {
Preconditions.checkNotNull(data);
readLock();
try {
ConcurrentNavigableMap<String, ContainerStatus> map = null;
if (prevKey.length() == 0) {
map = containerMap.tailMap(containerMap.firstKey(), true);
} else {
map = containerMap.tailMap(prevKey, false);
}
int currentCount = 0;
for (ContainerStatus entry : map.values()) {
if (currentCount < count) {
data.add(entry.getContainer());
currentCount++;
} else {
return;
}
}
} finally {
readUnlock();
}
}
/**
* Get metadata about a specific container.
*
* @param containerName - Name of the container
* @return ContainerData - Container Data.
* @throws IOException
*/
@Override
public ContainerData readContainer(String containerName) throws IOException {
return containerMap.get(containerName).getContainer();
}
/**
* Supports clean shutdown of container.
*
* @throws IOException
*/
@Override
public void shutdown() throws IOException {
}
@VisibleForTesting
ConcurrentSkipListMap<String, ContainerStatus> getContainerMap() {
return containerMap;
}
/**
* Acquire read lock.
*/
@Override
public void readLock() {
this.lock.readLock().lock();
}
/**
* Release read lock.
*/
@Override
public void readUnlock() {
this.lock.readLock().unlock();
}
/**
* Check if the current thread holds read lock.
*/
@Override
public boolean hasReadLock() {
return this.lock.readLock().tryLock();
}
/**
* Acquire write lock.
*/
@Override
public void writeLock() {
this.lock.writeLock().lock();
}
/**
* Acquire write lock, unless interrupted while waiting.
*/
@Override
public void writeLockInterruptibly() throws InterruptedException {
this.lock.writeLock().lockInterruptibly();
}
/**
* Release write lock.
*/
@Override
public void writeUnlock() {
this.lock.writeLock().unlock();
}
/**
* Check if the current thread holds write lock.
*/
@Override
public boolean hasWriteLock() {
return this.lock.writeLock().isHeldByCurrentThread();
}
/**
* Filter out only container files from the container metadata dir.
*/
private static class ContainerFilter implements FilenameFilter {
/**
* Tests if a specified file should be included in a file list.
*
* @param dir the directory in which the file was found.
* @param name the name of the file.
* @return <code>true</code> if and only if the name should be included in
* the file list; <code>false</code> otherwise.
*/
@Override
public boolean accept(File dir, String name) {
return name.endsWith(CONTAINER_EXTENSION);
}
}
/**
* This is an immutable class that represents the state of a container. if the
* container reading encountered an error when we boot up we will post that
* info to a recovery queue and keep the info in the containerMap.
* <p/>
* if and when the issue is fixed, the expectation is that this entry will be
* deleted by the recovery thread from the containerMap and will insert entry
* instead of modifying this class.
*/
@VisibleForTesting
static class ContainerStatus {
private final ContainerData containerData;
private final boolean active;
/**
* Creates a Container Status class.
*
* @param containerData - ContainerData.
* @param active - Active or not active.
*/
public ContainerStatus(ContainerData containerData, boolean active) {
this.containerData = containerData;
this.active = active;
}
/**
* Returns container if it is active. It is not active if we have had an
* error and we are waiting for the background threads to fix the issue.
*
* @return ContainerData.
*/
public ContainerData getContainer() {
if (active) {
return containerData;
}
return null;
}
/**
* Indicates if a container is Active.
*
* @return
*/
public boolean isActive() {
return active;
}
}
}

44
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManager.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.container.common.interfaces;
import java.io.IOException;
import java.nio.file.Path;
/**
* Returns physical path locations, where the containers will be created.
*/
public interface ContainerLocationManager {
/**
* Returns the path where the container should be placed from a set of
* locations.
*
* @return A path where we should place this container and metadata.
* @throws IOException
*/
Path getContainerPath() throws IOException;
/**
* Returns the path where the container Data file are stored.
*
* @return a path where we place the LevelDB and data files of a container.
* @throws IOException
*/
Path getDataPath(String containerName) throws IOException;
}

33
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
View File

@ -20,18 +20,35 @@ package org.apache.hadoop.ozone.container.common.interfaces;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.util.RwLock;
import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
import org.apache.hadoop.ozone.container.common.helpers.Pipeline;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
/**
* Interface for container operations.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public interface ContainerManager {
public interface ContainerManager extends RwLock {
/**
* Init call that sets up a container Manager.
*
* @param config - Configuration.
* @param containerDirs - List of Metadata Container locations.
* @param dataset - FSDataset.
* @throws IOException
*/
void init(Configuration config, List<Path> containerDirs,
FsDatasetSpi dataset)
throws IOException;
/**
* Creates a container with the given name.
@ -56,20 +73,28 @@ public interface ContainerManager {
/**
* As simple interface for container Iterations.
*
* @param start - Starting index
* @param prevKey - Starting KeyValue
* @param count - how many to return
* @param data - Actual containerData
* @throws IOException
*/
void listContainer(long start, long count, List<ContainerData> data)
void listContainer(String prevKey, long count, List<ContainerData> data)
throws IOException;
/**
* Get metadata about a specific container.
*
* @param containerName - Name of the container
* @return ContainerData
* @return ContainerData - Container Data.
* @throws IOException
*/
ContainerData readContainer(String containerName) throws IOException;
/**
* Supports clean shutdown of container.
*
* @throws IOException
*/
void shutdown() throws IOException;
}

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/client/XceiverClient.java
View File

@ -26,10 +26,10 @@ import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.logging.LogLevel;
import io.netty.handler.logging.LoggingHandler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.container.common.helpers.Pipeline;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -42,7 +42,7 @@ import java.io.IOException;
public class XceiverClient {
static final Logger LOG = LoggerFactory.getLogger(XceiverClient.class);
private final Pipeline pipeline;
private final OzoneConfiguration config;
private final Configuration config;
private ChannelFuture channelFuture;
private Bootstrap b;
private EventLoopGroup group;
@ -53,7 +53,7 @@ public class XceiverClient {
* @param pipeline - Pipeline that defines the machines.
* @param config -- Ozone Config
*/
public XceiverClient(Pipeline pipeline, OzoneConfiguration config) {
public XceiverClient(Pipeline pipeline, Configuration config) {
Preconditions.checkNotNull(pipeline);
Preconditions.checkNotNull(config);
this.pipeline = pipeline;

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java
View File

@ -26,8 +26,8 @@ import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.logging.LogLevel;
import io.netty.handler.logging.LoggingHandler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
/**
@ -47,7 +47,7 @@ public final class XceiverServer {
*
* @param conf - Configuration
*/
public XceiverServer(OzoneConfiguration conf,
public XceiverServer(Configuration conf,
ContainerDispatcher dispatcher) {
Preconditions.checkNotNull(conf);
this.port = conf.getInt(OzoneConfigKeys.DFS_OZONE_CONTAINER_IPC_PORT,

52
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneLevelDBStore.java → hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/LevelDBStore.java
View File

@ -16,43 +16,45 @@
* limitations under the License.
*/
package org.apache.hadoop.ozone.web.localstorage;
package org.apache.hadoop.ozone.container.common.utils;
import org.fusesource.leveldbjni.JniDBFactory;
import org.iq80.leveldb.DB;
import org.iq80.leveldb.DBIterator;
import org.iq80.leveldb.Options;
import java.io.File;
import java.io.IOException;
/**
* OzoneLevelDBStore is used by the local
* OzoneStore which is used in testing.
* LevelDB interface.
*/
class OzoneLevelDBStore {
public class LevelDBStore {
private DB db;
private final File dbFile;
/**
* Opens a DB file.
*
* @param dbPath - DB File path
* @param dbPath - DB File path
* @param createIfMissing - Create if missing
*
* @throws IOException
*/
OzoneLevelDBStore(File dbPath, boolean createIfMissing) throws IOException {
public LevelDBStore(File dbPath, boolean createIfMissing) throws
IOException {
Options options = new Options();
options.createIfMissing(createIfMissing);
db = JniDBFactory.factory.open(dbPath, options);
if (db == null) {
throw new IOException("Db is null");
}
this.dbFile = dbPath;
}
/**
* Puts a Key into file.
*
* @param key - key
* @param key - key
* @param value - value
*/
public void put(byte[] key, byte[] value) {
@ -63,7 +65,6 @@ class OzoneLevelDBStore {
* Get Key.
*
* @param key key
*
* @return value
*/
public byte[] get(byte[] key) {
@ -87,4 +88,37 @@ class OzoneLevelDBStore {
public void close() throws IOException {
db.close();
}
/**
* Returns true if the DB is empty.
*
* @return boolean
* @throws IOException
*/
public boolean isEmpty() throws IOException {
DBIterator iter = db.iterator();
try {
iter.seekToFirst();
return iter.hasNext();
} finally {
iter.close();
}
}
/**
* Returns Java File Object that points to the DB.
* @return File
*/
public File getDbFile() {
return dbFile;
}
/**
* Returns the actual levelDB object.
* @return DB handle.
*/
public DB getDB() {
return db;
}
}

121
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java Normal file
View File

@ -0,0 +1,121 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.container.ozoneimpl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.impl.ContainerManagerImpl;
import org.apache.hadoop.ozone.container.common.impl.Dispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
import org.apache.hadoop.ozone.container.common.transport.server.XceiverServer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.LinkedList;
import java.util.List;
import java.nio.file.Path;
/**
* Ozone main class sets up the network server and initializes the container
* layer.
*/
public class OzoneContainer {
private static final Logger LOG =
LoggerFactory.getLogger(OzoneContainer.class);
private final Configuration ozoneConfig;
private final FsDatasetSpi dataSet;
private final ContainerDispatcher dispatcher;
private final ContainerManager manager;
private final XceiverServer server;
/**
* Creates a network endpoint and enables Ozone container.
*
* @param ozoneConfig - Config
* @param dataSet - FsDataset.
* @throws IOException
*/
public OzoneContainer(Configuration ozoneConfig, FsDatasetSpi dataSet) throws
Exception {
List<Path> locations = new LinkedList<>();
String[] paths = ozoneConfig.getStrings(OzoneConfigKeys
.DFS_OZONE_METADATA_DIRS);
if (paths != null && paths.length > 0) {
for (String p : paths) {
locations.add(Paths.get(p));
}
} else {
getDataDir(dataSet, locations);
}
this.ozoneConfig = ozoneConfig;
this.dataSet = dataSet;
manager = new ContainerManagerImpl();
manager.init(this.ozoneConfig, locations, this.dataSet);
this.dispatcher = new Dispatcher(manager);
server = new XceiverServer(this.ozoneConfig, this.dispatcher);
}
/**
* Starts serving requests to ozone container.
* @throws Exception
*/
public void start() throws Exception {
server.start();
}
/**
* Stops the ozone container.
* @throws Exception
*/
public void stop() throws Exception {
server.stop();
}
/**
* Returns a paths to data dirs.
* @param dataset - FSDataset.
* @param pathList - List of paths.
* @throws IOException
*/
private void getDataDir(FsDatasetSpi dataset, List<Path> pathList) throws
IOException {
FsDatasetSpi.FsVolumeReferences references;
try {
synchronized (dataset) {
references = dataset.getFsVolumeReferences();
for (int ndx = 0; ndx < references.size(); ndx++) {
FsVolumeSpi vol = references.get(ndx);
pathList.add(Paths.get(vol.getBasePath()));
}
references.close();
}
} catch (IOException ex) {
LOG.error("Unable to get volume paths.", ex);
throw new IOException("Internal error", ex);
}
}
}

21
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/package-info.java Normal file
View File

@ -0,0 +1,21 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.container.ozoneimpl;
/**
Ozone main that calls into the container layer
**/

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
View File

@ -26,7 +26,7 @@ import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.KeyInfo;
import org.apache.hadoop.ozone.web.response.ListKeys;
import org.apache.hadoop.ozone.web.utils.OzoneConsts;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.http.HttpEntity;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
View File

@ -25,7 +25,7 @@ import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.ListBuckets;
import org.apache.hadoop.ozone.web.response.VolumeInfo;
import org.apache.hadoop.ozone.web.utils.OzoneConsts;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
View File

@ -19,7 +19,7 @@ package org.apache.hadoop.ozone.web.handlers;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.web.utils.OzoneConsts;
import org.apache.hadoop.ozone.OzoneConsts;
import java.util.LinkedList;
import java.util.List;

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
View File

@ -25,7 +25,7 @@ import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.headers.Header;
import org.apache.hadoop.ozone.web.interfaces.Bucket;
import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
import org.apache.hadoop.ozone.web.utils.OzoneConsts;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.slf4j.MDC;
@ -37,7 +37,7 @@ import java.io.IOException;
import static java.net.HttpURLConnection.HTTP_CREATED;
import static java.net.HttpURLConnection.HTTP_OK;
import static org.apache.hadoop.ozone.web.utils.OzoneConsts.OZONE_FUNCTION;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_FUNCTION;
/**

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketProcessTemplate.java
View File

@ -27,7 +27,7 @@ import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
import org.apache.hadoop.ozone.web.interfaces.UserAuth;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.ListKeys;
import org.apache.hadoop.ozone.web.utils.OzoneConsts;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.slf4j.Logger;
@ -47,10 +47,11 @@ import java.util.LinkedList;
import java.util.List;
import static java.net.HttpURLConnection.HTTP_OK;
import static org.apache.hadoop.ozone.web.utils.OzoneConsts.OZONE_COMPONENT;
import static org.apache.hadoop.ozone.web.utils.OzoneConsts.OZONE_RESOURCE;
import static org.apache.hadoop.ozone.web.utils.OzoneConsts.OZONE_REQUEST;
import static org.apache.hadoop.ozone.web.utils.OzoneConsts.OZONE_USER;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_COMPONENT;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RESOURCE;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_REQUEST;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_USER;

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyProcessTemplate.java
View File

@ -44,10 +44,11 @@ import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.INVALID_BUCKET_N
import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.INVALID_REQUEST;
import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.SERVER_ERROR;
import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.newError;
import static org.apache.hadoop.ozone.web.utils.OzoneConsts.OZONE_COMPONENT;
import static org.apache.hadoop.ozone.web.utils.OzoneConsts.OZONE_REQUEST;
import static org.apache.hadoop.ozone.web.utils.OzoneConsts.OZONE_RESOURCE;
import static org.apache.hadoop.ozone.web.utils.OzoneConsts.OZONE_USER;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_COMPONENT;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RESOURCE;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_REQUEST;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_USER;
/**
* This class abstracts way the repetitive tasks in Key handling code.

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeHandler.java
View File

@ -38,7 +38,7 @@ import java.io.IOException;
import static java.net.HttpURLConnection.HTTP_CREATED;
import static java.net.HttpURLConnection.HTTP_OK;
import static org.apache.hadoop.ozone.web.utils.OzoneConsts.OZONE_FUNCTION;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_FUNCTION;
/**
* VolumeHandler handles volume specific HTTP calls.

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeProcessTemplate.java
View File

@ -41,10 +41,10 @@ import java.nio.file.FileAlreadyExistsException;
import java.nio.file.NoSuchFileException;
import static java.net.HttpURLConnection.HTTP_OK;
import static org.apache.hadoop.ozone.web.utils.OzoneConsts.OZONE_COMPONENT;
import static org.apache.hadoop.ozone.web.utils.OzoneConsts.OZONE_RESOURCE;
import static org.apache.hadoop.ozone.web.utils.OzoneConsts.OZONE_REQUEST;
import static org.apache.hadoop.ozone.web.utils.OzoneConsts.OZONE_USER;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_COMPONENT;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RESOURCE;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_REQUEST;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_USER;
/**

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java
View File

@ -25,6 +25,7 @@ import org.slf4j.LoggerFactory;
import org.slf4j.Logger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.utils.LevelDBStore;
import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.handlers.BucketArgs;
@ -37,7 +38,7 @@ import org.apache.hadoop.ozone.web.response.ListBuckets;
import org.apache.hadoop.ozone.web.response.ListVolumes;
import org.apache.hadoop.ozone.web.response.VolumeInfo;
import org.apache.hadoop.ozone.web.response.VolumeOwner;
import org.apache.hadoop.ozone.web.utils.OzoneConsts;
import org.apache.hadoop.ozone.OzoneConsts;
import org.iq80.leveldb.DBException;
import org.apache.commons.codec.digest.DigestUtils;
@ -125,8 +126,8 @@ public final class OzoneMetadataManager {
private static final String USER_DB = "/user.db";
private static final String META_DB = "/metadata.db";
private static OzoneMetadataManager bm = null;
private OzoneLevelDBStore userDB;
private OzoneLevelDBStore metadataDB;
private LevelDBStore userDB;
private LevelDBStore metadataDB;
private ReadWriteLock lock;
private Charset encoding = Charset.forName("UTF-8");
private String storageRoot;
@ -154,8 +155,8 @@ public final class OzoneMetadataManager {
}
try {
userDB = new OzoneLevelDBStore(new File(storageRoot + USER_DB), true);
metadataDB = new OzoneLevelDBStore(new File(storageRoot + META_DB), true);
userDB = new LevelDBStore(new File(storageRoot + USER_DB), true);
metadataDB = new LevelDBStore(new File(storageRoot + META_DB), true);
inProgressObjects = new ConcurrentHashMap<>();
} catch (IOException ex) {
LOG.error("Cannot open db :" + ex.getMessage());

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/OzoneAcl.java
View File

@ -19,7 +19,7 @@
package org.apache.hadoop.ozone.web.request;
import org.apache.hadoop.ozone.web.utils.OzoneConsts;
import org.apache.hadoop.ozone.OzoneConsts;
import java.util.Objects;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java
View File

@ -20,7 +20,7 @@ package org.apache.hadoop.ozone.web.response;
import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.web.utils.OzoneConsts;
import org.apache.hadoop.ozone.OzoneConsts;
import org.codehaus.jackson.annotate.JsonAutoDetect;
import org.codehaus.jackson.annotate.JsonMethod;
import org.codehaus.jackson.map.ObjectMapper;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/Simple.java
View File

@ -24,7 +24,7 @@ import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.handlers.UserArgs;
import org.apache.hadoop.ozone.web.headers.Header;
import org.apache.hadoop.ozone.web.interfaces.UserAuth;
import org.apache.hadoop.ozone.web.utils.OzoneConsts;
import org.apache.hadoop.ozone.OzoneConsts;
import javax.ws.rs.core.HttpHeaders;
import java.util.List;

1
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.web.utils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream;
import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;

8
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeContainerProtocol.proto
View File

@ -168,7 +168,13 @@ message KeyValue {
message ContainerData {
required string name = 1;
repeated KeyValue metadata = 2;
optional string containerPath = 3;
optional string dbPath = 3;
optional string containerPath = 4;
}
message ContainerMeta {
required string fileName = 1;
required string hash = 2;
}
// Container Messages.

256
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java Normal file
View File

@ -0,0 +1,256 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.container.common.impl;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.utils.LevelDBStore;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import static org.apache.hadoop.ozone.container.ContainerTestHelper
.createSingleNodePipeline;
import static org.junit.Assert.fail;
/**
* Simple tests to verify that container persistence works as expected.
*/
public class TestContainerPersistence {
static String path;
static ContainerManagerImpl containerManager;
static OzoneConfiguration conf;
static FsDatasetSpi fsDataSet;
static MiniDFSCluster cluster;
static List<Path> pathLists = new LinkedList<>();
@BeforeClass
public static void init() throws IOException {
conf = new OzoneConfiguration();
URL p = conf.getClass().getResource("");
path = p.getPath().concat(
TestContainerPersistence.class.getSimpleName());
path += conf.getTrimmed(OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT,
OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT, path);
conf.setBoolean(OzoneConfigKeys.DFS_OBJECTSTORE_ENABLED_KEY, true);
conf.set(OzoneConfigKeys.DFS_STORAGE_HANDLER_TYPE_KEY, "local");
File containerDir = new File(path);
if (containerDir.exists()) {
FileUtils.deleteDirectory(new File(path));
}
Assert.assertTrue(containerDir.mkdirs());
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
fsDataSet = cluster.getDataNodes().get(0).getFSDataset();
containerManager = new ContainerManagerImpl();
}
@AfterClass
public static void shutdown() throws IOException {
cluster.shutdown();
FileUtils.deleteDirectory(new File(path));
}
@Before
public void setupPaths() throws IOException {
if (!new File(path).exists()) {
new File(path).mkdirs();
}
pathLists.clear();
containerManager.getContainerMap().clear();
pathLists.add(Paths.get(path));
containerManager.init(conf, pathLists, fsDataSet);
}
@After
public void cleanupDir() throws IOException {
FileUtils.deleteDirectory(new File(path));
}
@Test
public void testCreateContainer() throws Exception {
String containerName = OzoneUtils.getRequestID();
ContainerData data = new ContainerData(containerName);
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
containerManager.createContainer(createSingleNodePipeline(), data);
Assert.assertTrue(containerManager.getContainerMap()
.containsKey(containerName));
ContainerManagerImpl.ContainerStatus status = containerManager
.getContainerMap().get(containerName);
Assert.assertTrue(status.isActive());
Assert.assertNotNull(status.getContainer().getContainerPath());
Assert.assertNotNull(status.getContainer().getDBPath());
Assert.assertTrue(new File(status.getContainer().getContainerPath())
.exists());
String containerPathString = ContainerUtils.getContainerNameFromFile(new
File(status.getContainer().getContainerPath()));
Path meta = Paths.get(containerPathString);
String metadataFile = meta.toString() + OzoneConsts.CONTAINER_META;
Assert.assertTrue(new File(metadataFile).exists());
String dbPath = status.getContainer().getDBPath();
LevelDBStore store = null;
try {
store = new LevelDBStore(new File(dbPath), false);
Assert.assertNotNull(store.getDB());
} finally {
if (store != null) {
store.close();
}
}
}
@Test
public void testCreateDuplicateContainer() throws Exception {
String containerName = OzoneUtils.getRequestID();
ContainerData data = new ContainerData(containerName);
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
containerManager.createContainer(createSingleNodePipeline(), data);
try {
containerManager.createContainer(createSingleNodePipeline(), data);
fail("Expected Exception not thrown.");
} catch (IOException ex) {
Assert.assertNotNull(ex);
}
}
@Test
public void testDeleteContainer() throws Exception {
String containerName1 = OzoneUtils.getRequestID();
String containerName2 = OzoneUtils.getRequestID();
ContainerData data = new ContainerData(containerName1);
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
containerManager.createContainer(createSingleNodePipeline(), data);
data = new ContainerData(containerName2);
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
containerManager.createContainer(createSingleNodePipeline(), data);
Assert.assertTrue(containerManager.getContainerMap()
.containsKey(containerName1));
Assert.assertTrue(containerManager.getContainerMap()
.containsKey(containerName2));
containerManager.deleteContainer(createSingleNodePipeline(),
containerName1);
Assert.assertFalse(containerManager.getContainerMap()
.containsKey(containerName1));
// Let us make sure that we are able to re-use a container name after
// delete.
data = new ContainerData(containerName1);
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
containerManager.createContainer(createSingleNodePipeline(), data);
// Assert we still have both containers.
Assert.assertTrue(containerManager.getContainerMap()
.containsKey(containerName1));
Assert.assertTrue(containerManager.getContainerMap()
.containsKey(containerName2));
}
/**
* This test creates 1000 containers and reads them back 5 containers at a
* time and verifies that we did get back all containers.
*
* @throws IOException
*/
@Test
public void testListContainer() throws IOException {
final int count = 1000;
final int step = 5;
Map<String, ContainerData> testMap = new HashMap<>();
for (int x = 0; x < count; x++) {
String containerName = OzoneUtils.getRequestID();
ContainerData data = new ContainerData(containerName);
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
containerManager.createContainer(createSingleNodePipeline(), data);
testMap.put(containerName, data);
}
int counter = 0;
String prevKey = "";
List<ContainerData> results = new LinkedList<>();
while (counter < count) {
containerManager.listContainer(prevKey, step, results);
for (int y = 0; y < results.size(); y++) {
testMap.remove(results.get(y).getContainerName());
}
counter += step;
String nextKey = results.get(results.size() - 1).getContainerName();
//Assert that container is returning results in a sorted fashion.
Assert.assertTrue(prevKey.compareTo(nextKey) < 0);
prevKey = nextKey;
results.clear();
}
// Assert that we listed all the keys that we had put into
// container.
Assert.assertTrue(testMap.isEmpty());
}
}

118
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java Normal file
View File

@ -0,0 +1,118 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.container.ozoneimpl;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
import org.apache.hadoop.ozone.container.common.helpers.Pipeline;
import org.apache.hadoop.ozone.container.common.impl.ContainerManagerImpl;
import org.apache.hadoop.ozone.container.common.transport.client.XceiverClient;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.net.URL;
public class TestOzoneContainer {
@Test
public void testCreateOzoneContainer() throws Exception {
Configuration conf = new OzoneConfiguration();
URL p = conf.getClass().getResource("");
String path = p.getPath().concat(
TestOzoneContainer.class.getSimpleName());
path += conf.getTrimmed(OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT,
OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT, path);
// We don't start Ozone Container via data node, we will do it
// independently in our test path.
conf.setBoolean(OzoneConfigKeys.DFS_OBJECTSTORE_ENABLED_KEY, false);
conf.set(OzoneConfigKeys.DFS_STORAGE_HANDLER_TYPE_KEY, "local");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline();
conf.setInt(OzoneConfigKeys.DFS_OZONE_CONTAINER_IPC_PORT,
pipeline.getLeader().getContainerPort());
OzoneContainer container = new OzoneContainer(conf, cluster.getDataNodes
().get(0).getFSDataset());
container.start();
XceiverClient client = new XceiverClient(pipeline, conf);
client.connect();
ContainerProtos.ContainerCommandRequestProto request =
ContainerTestHelper.getCreateContainerRequest();
ContainerProtos.ContainerCommandResponseProto response =
client.sendCommand(request);
Assert.assertNotNull(response);
Assert.assertTrue(request.getTraceID().equals(response.getTraceID()));
container.stop();
cluster.shutdown();
}
@Test
public void testOzoneContainerViaDataNode() throws Exception {
Configuration conf = new OzoneConfiguration();
URL p = conf.getClass().getResource("");
String path = p.getPath().concat(
TestOzoneContainer.class.getSimpleName());
path += conf.getTrimmed(OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT,
OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT, path);
// Start ozone container Via Datanode create.
conf.setBoolean(OzoneConfigKeys.DFS_OBJECTSTORE_ENABLED_KEY, true);
conf.set(OzoneConfigKeys.DFS_STORAGE_HANDLER_TYPE_KEY, "local");
Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline();
conf.setInt(OzoneConfigKeys.DFS_OZONE_CONTAINER_IPC_PORT,
pipeline.getLeader().getContainerPort());
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
// This client talks to ozone container via datanode.
XceiverClient client = new XceiverClient(pipeline, conf);
client.connect();
ContainerProtos.ContainerCommandRequestProto request =
ContainerTestHelper.getCreateContainerRequest();
ContainerProtos.ContainerCommandResponseProto response =
client.sendCommand(request);
Assert.assertNotNull(response);
Assert.assertTrue(request.getTraceID().equals(response.getTraceID()));
cluster.shutdown();
}
}

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestBucketInfo.java
View File

@ -22,7 +22,7 @@ package org.apache.hadoop.ozone.web;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.utils.OzoneConsts;
import org.apache.hadoop.ozone.OzoneConsts;
import org.junit.Test;
import java.io.IOException;

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneVolumes.java
View File

@ -23,7 +23,7 @@ import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
import org.apache.hadoop.ozone.web.headers.Header;
import org.apache.hadoop.ozone.web.utils.OzoneConsts;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.util.Time;
import org.apache.http.HttpResponse;

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneWebAccess.java
View File

@ -29,13 +29,12 @@ import java.util.Date;
import java.util.Locale;
import javax.ws.rs.core.HttpHeaders;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.web.headers.Header;
import org.apache.hadoop.ozone.web.utils.OzoneConsts;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.util.Time;
import org.apache.http.HttpResponse;

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
View File

@ -24,12 +24,11 @@ import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.utils.OzoneConsts;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;