HDDS-359. RocksDB Profiles support. Contributed by Anu Engineer.

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java

@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hdds;
 
+import org.apache.hadoop.utils.db.DBProfile;
+
 /**
  * This class contains constants for configuration keys and default values
  * used in hdds.
@@ -58,4 +60,8 @@ public final class HddsConfigKeys {
   public static final String HDDS_DATANODE_VOLUME_CHOOSING_POLICY =
       "hdds.datanode.volume.choosing.policy";
 
+  // DB Profiles used by ROCKDB instances.
+  public static final String HDDS_DB_PROFILE = "hdds.db.profile";
+  public static final DBProfile HDDS_DEFAULT_DB_PROFILE = DBProfile.SSD;
+
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBConfigFromFile.java

@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.utils.db;
+
+import com.google.common.base.Preconditions;
+import org.eclipse.jetty.util.StringUtil;
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.DBOptions;
+import org.rocksdb.Env;
+import org.rocksdb.OptionsUtil;
+import org.rocksdb.RocksDBException;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+
+/**
+ * A Class that controls the standard config options of RocksDB.
+ * <p>
+ * Important : Some of the functions in this file are magic functions designed
+ * for the use of OZONE developers only. Due to that this information is
+ * documented in this files only and is *not* intended for end user consumption.
+ * Please do not use this information to tune your production environments.
+ * Please remember the SpiderMan principal; with great power comes great
+ * responsibility.
+ */
+public final class DBConfigFromFile {
+  public static final String CONFIG_DIR = "HADOOP_CONF_DIR";
+
+  private DBConfigFromFile() {
+  }
+
+  public static File getConfigLocation() throws IOException {
+    String path = System.getenv(CONFIG_DIR);
+
+    // Make testing easy.
+    // If there is No Env. defined, let us try to read the JVM property
+    if (StringUtil.isBlank(path)) {
+      path = System.getProperty(CONFIG_DIR);
+    }
+
+    if (StringUtil.isBlank(path)) {
+      throw new IOException("Unable to find the configuration directory. "
+          + "Please make sure that HADOOP_CONF_DIR is setup correctly ");
+    }
+    return new File(path);
+
+  }
+
+  /**
+   * This class establishes a magic pattern where we look for DBFile.ini as the
+   * options for RocksDB.
+   *
+   * @param dbFileName - The DBFile Name. For example, OzoneManager.db
+   * @return Name of the DB File options
+   */
+  public static String getOptionsFileNameFromDB(String dbFileName) {
+    Preconditions.checkNotNull(dbFileName);
+    return dbFileName + ".ini";
+  }
+
+  /**
+   * One of the Magic functions designed for the use of Ozone Developers *ONLY*.
+   * This function takes the name of DB file and looks up the a .ini file that
+   * follows the ROCKSDB config format and uses that file for DBOptions and
+   * Column family Options. The Format for this file is specified by RockDB.
+   * <p>
+   * Here is a sample config from RocksDB sample Repo.
+   * <p>
+   * https://github.com/facebook/rocksdb/blob/master/examples
+   * /rocksdb_option_file_example.ini
+   * <p>
+   * We look for a specific pattern, say OzoneManager.db will have its configs
+   * specified in OzoneManager.db.ini. This option is used only by the
+   * performance testing group to allow tuning of all parameters freely.
+   * <p>
+   * For the end users we offer a set of Predefined options that is easy to use
+   * and the user does not need to become an expert in RockDB config.
+   * <p>
+   * This code assumes the .ini file is placed in the same directory as normal
+   * config files. That is in $HADOOP_DIR/etc/hadoop. For example, if we want to
+   * control OzoneManager.db configs from a file, we need to create a file
+   * called OzoneManager.db.ini and place that file in $HADOOP_DIR/etc/hadoop.
+   *
+   * @param dbFileName - The DB File Name, for example, OzoneManager.db.
+   * @param cfDescs - ColumnFamily Handles.
+   * @return DBOptions, Options to be used for opening/creating the DB.
+   * @throws IOException
+   */
+  public static DBOptions readFromFile(String dbFileName,
+      List<ColumnFamilyDescriptor> cfDescs) throws IOException {
+    Preconditions.checkNotNull(dbFileName);
+    Preconditions.checkNotNull(cfDescs);
+    Preconditions.checkArgument(cfDescs.size() > 0);
+
+    //TODO: Add Documentation on how to support RocksDB Mem Env.
+    Env env = Env.getDefault();
+    DBOptions options = null;
+    Path optionsFile = Paths.get(getConfigLocation().toString(),
+        getOptionsFileNameFromDB(dbFileName));
+
+    if (optionsFile.toFile().exists()) {
+      options = new DBOptions();
+      try {
+        OptionsUtil.loadOptionsFromFile(optionsFile.toString(),
+            env, options, cfDescs, true);
+
+      } catch (RocksDBException rdEx) {
+        RDBTable.toIOException("Unable to find/open Options file.", rdEx);
+      }
+    }
+    return options;
+  }
+
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBProfile.java

@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.utils.db;
+
+import org.apache.hadoop.conf.StorageUnit;
+import org.rocksdb.BlockBasedTableConfig;
+import org.rocksdb.BloomFilter;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.CompactionStyle;
+import org.rocksdb.DBOptions;
+
+import java.math.BigDecimal;
+
+/**
+ * User visible configs based RocksDB tuning page. Documentation for Options.
+ * <p>
+ * https://github.com/facebook/rocksdb/blob/master/include/rocksdb/options.h
+ * <p>
+ * Most tuning parameters are based on this URL.
+ * <p>
+ * https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning
+ */
+public enum DBProfile {
+  //TODO : Add more profiles like TEST etc.
+  SSD {
+    @Override
+    public String toString() {
+      return "DBProfile.SSD";
+    }
+
+    @Override
+    public ColumnFamilyOptions getColumnFamilyOptions() {
+
+      // Set BlockCacheSize to 256 MB. This should not be an issue for HADOOP.
+      final long blockCacheSize = toLong(StorageUnit.MB.toBytes(256.00));
+
+      // Set the Default block size to 16KB
+      final long blockSize = toLong(StorageUnit.KB.toBytes(16));
+
+      // Write Buffer Size -- set to 128 MB
+      final long writeBufferSize = toLong(StorageUnit.MB.toBytes(128));
+
+      return new ColumnFamilyOptions()
+          .setLevelCompactionDynamicLevelBytes(true)
+          .setWriteBufferSize(writeBufferSize)
+          .setTableFormatConfig(
+              new BlockBasedTableConfig()
+                  .setBlockCacheSize(blockCacheSize)
+                  .setBlockSize(blockSize)
+                  .setCacheIndexAndFilterBlocks(true)
+                  .setPinL0FilterAndIndexBlocksInCache(true)
+                  .setFilter(new BloomFilter()));
+    }
+
+    @Override
+    public DBOptions getDBOptions() {
+      final int maxBackgroundCompactions = 4;
+      final int maxBackgroundFlushes = 2;
+      final long bytesPerSync = toLong(StorageUnit.MB.toBytes(1.00));
+      final boolean createIfMissing = true;
+      final boolean createMissingColumnFamilies = true;
+      return new DBOptions()
+          .setIncreaseParallelism(Runtime.getRuntime().availableProcessors())
+          .setMaxBackgroundCompactions(maxBackgroundCompactions)
+          .setMaxBackgroundFlushes(maxBackgroundFlushes)
+          .setBytesPerSync(bytesPerSync)
+          .setCreateIfMissing(createIfMissing)
+          .setCreateMissingColumnFamilies(createMissingColumnFamilies);
+    }
+
+
+  },
+  DISK {
+    @Override
+    public String toString() {
+      return "DBProfile.DISK";
+    }
+
+    @Override
+    public DBOptions getDBOptions() {
+      final long readAheadSize = toLong(StorageUnit.MB.toBytes(4.00));
+      return SSD.getDBOptions().setCompactionReadaheadSize(readAheadSize);
+    }
+
+    @Override
+    public ColumnFamilyOptions getColumnFamilyOptions() {
+      ColumnFamilyOptions columnFamilyOptions = SSD.getColumnFamilyOptions();
+      columnFamilyOptions.setCompactionStyle(CompactionStyle.LEVEL);
+      return columnFamilyOptions;
+    }
+
+
+  };
+
+  private static long toLong(double value) {
+    BigDecimal temp = new BigDecimal(value);
+    return temp.longValue();
+  }
+
+  public abstract DBOptions getDBOptions();
+
+  public abstract ColumnFamilyOptions getColumnFamilyOptions();
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStoreBuilder.java

@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.utils.db;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.eclipse.jetty.util.StringUtil;
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.DBOptions;
+import org.rocksdb.RocksDB;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_DB_PROFILE;
+
+/**
+ * DBStore Builder.
+ */
+public final class DBStoreBuilder {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(DBStoreBuilder.class);
+  private Set<TableConfig> tables;
+  private DBProfile dbProfile;
+  private DBOptions rocksDBOption;
+  private String dbname;
+  private Path dbPath;
+  private List<String> tableNames;
+  private Configuration configuration;
+
+  private DBStoreBuilder(Configuration configuration) {
+    tables = new HashSet<>();
+    tableNames = new LinkedList<>();
+    this.configuration = configuration;
+  }
+
+  public static DBStoreBuilder newBuilder(Configuration configuration) {
+    return new DBStoreBuilder(configuration);
+  }
+
+  public DBStoreBuilder setProfile(DBProfile profile) {
+    dbProfile = profile;
+    return this;
+  }
+
+  public DBStoreBuilder setName(String name) {
+    dbname = name;
+    return this;
+  }
+
+  public DBStoreBuilder addTable(String tableName) {
+    tableNames.add(tableName);
+    return this;
+  }
+
+  public DBStoreBuilder addTable(String tableName, ColumnFamilyOptions option)
+      throws IOException {
+    TableConfig tableConfig = new TableConfig(tableName, option);
+    if (!tables.add(tableConfig)) {
+      String message = "Unable to add the table: " + tableName +
+          ".  Please check if this table name is already in use.";
+      LOG.error(message);
+      throw new IOException(message);
+    }
+    LOG.info("using custom profile for table: {}", tableName);
+    return this;
+  }
+
+  public DBStoreBuilder setDBOption(DBOptions option) {
+    rocksDBOption = option;
+    return this;
+  }
+
+  public DBStoreBuilder setPath(Path path) {
+    Preconditions.checkNotNull(path);
+    dbPath = path;
+    return this;
+  }
+
+  /**
+   * Builds a DBStore instance and returns that.
+   *
+   * @return DBStore
+   */
+  public DBStore build() throws IOException {
+    if(StringUtil.isBlank(dbname) || (dbPath == null)) {
+      LOG.error("Required Parameter missing.");
+      throw new IOException("Required parameter is missing. Please make sure "
+          + "sure Path and DB name is provided.");
+    }
+    processDBProfile();
+    processTables();
+    DBOptions options = getDbProfile();
+    File dbFile = getDBFile();
+    if (!dbFile.getParentFile().exists()) {
+      throw new IOException("The DB destination directory should exist.");
+    }
+    return new RDBStore(dbFile, options, tables);
+  }
+
+  /**
+   * if the DBProfile is not set, we will default to using default from the
+   * config file.
+   */
+  private void processDBProfile() {
+    if (dbProfile == null) {
+      dbProfile = this.configuration.getEnum(HDDS_DB_PROFILE,
+          HDDS_DEFAULT_DB_PROFILE);
+    }
+  }
+
+  private void processTables() throws IOException {
+    if (tableNames.size() > 0) {
+      for (String name : tableNames) {
+        addTable(name, dbProfile.getColumnFamilyOptions());
+        LOG.info("Using default column profile:{} for Table:{}",
+            dbProfile.toString(), name);
+      }
+    }
+    addTable(DFSUtil.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY),
+        dbProfile.getColumnFamilyOptions());
+    LOG.info("Using default column profile:{} for Table:{}",
+        dbProfile.toString(),
+        DFSUtil.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY));
+  }
+
+  private DBOptions getDbProfile() {
+    if (rocksDBOption != null) {
+      return rocksDBOption;
+    }
+    DBOptions option = null;
+    if (StringUtil.isNotBlank(dbname)) {
+      List<ColumnFamilyDescriptor> columnFamilyDescriptors = new LinkedList<>();
+
+      for (TableConfig tc : tables) {
+        columnFamilyDescriptors.add(tc.getDescriptor());
+      }
+
+      if (columnFamilyDescriptors.size() > 0) {
+        try {
+          option = DBConfigFromFile.readFromFile(dbname,
+              columnFamilyDescriptors);
+          if(option != null) {
+            LOG.info("Using Configs from {}.ini file", dbname);
+          }
+        } catch (IOException ex) {
+          LOG.info("Unable to read ROCKDB config", ex);
+        }
+      }
+    }
+
+    if (option == null) {
+      LOG.info("Using default options. {}", dbProfile.toString());
+      return dbProfile.getDBOptions();
+    }
+    return option;
+  }
+
+  private File getDBFile() throws IOException {
+    if (dbPath == null) {
+      LOG.error("DB path is required.");
+      throw new IOException("A Path to for DB file is needed.");
+    }
+
+    if (StringUtil.isBlank(dbname)) {
+      LOG.error("DBName is a required.");
+      throw new IOException("A valid DB name is required.");
+    }
+    return Paths.get(dbPath.toString(), dbname).toFile();
+  }
+
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java

@@ -26,7 +26,7 @@ import org.apache.hadoop.utils.RocksDBStoreMBean;
 import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting;
 import org.rocksdb.ColumnFamilyDescriptor;
 import org.rocksdb.ColumnFamilyHandle;
-import org.rocksdb.ColumnFamilyOptions;
+
 import org.rocksdb.DBOptions;
 import org.rocksdb.RocksDB;
 import org.rocksdb.RocksDBException;
@@ -38,12 +38,12 @@ import org.slf4j.LoggerFactory;
 import javax.management.ObjectName;
 import java.io.File;
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Hashtable;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * RocksDB Store that supports creating Tables in DB.
@@ -58,7 +58,7 @@ public class RDBStore implements DBStore {
   private final Hashtable<String, ColumnFamilyHandle> handleTable;
   private ObjectName statMBeanName;
 
-  public RDBStore(File dbFile, DBOptions options, List<String> families)
+  public RDBStore(File dbFile, DBOptions options, Set<TableConfig> families)
       throws IOException {
     Preconditions.checkNotNull(dbFile, "DB file location cannot be null");
     Preconditions.checkNotNull(families);
@@ -69,10 +69,8 @@ public class RDBStore implements DBStore {
         new ArrayList<>();
     final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
 
-    for (String family : families) {
-      columnFamilyDescriptors.add(
-          new ColumnFamilyDescriptor(family.getBytes(StandardCharsets.UTF_8),
-              new ColumnFamilyOptions()));
+    for (TableConfig family : families) {
+      columnFamilyDescriptors.add(family.getDescriptor());
     }
 
     dbOptions = options;
@@ -141,19 +139,23 @@ public class RDBStore implements DBStore {
     for (final ColumnFamilyHandle handle : handleTable.values()) {
       handle.close();
     }
-    if (dbOptions != null) {
-      dbOptions.close();
-    }
-    if (writeOptions != null) {
-      writeOptions.close();
-    }
+
     if (statMBeanName != null) {
       MBeans.unregister(statMBeanName);
       statMBeanName = null;
     }
+
     if (db != null) {
       db.close();
     }
+
+    if (dbOptions != null) {
+      dbOptions.close();
+    }
+
+    if (writeOptions != null) {
+      writeOptions.close();
+    }
   }
 
   @Override
@@ -221,7 +223,7 @@ public class RDBStore implements DBStore {
   @Override
   public long getEstimatedKeyCount() throws IOException {
     try {
-      return Long.parseLong(db.getProperty("rocksdb.estimate-num-keys"));
+      return db.getLongProperty("rocksdb.estimate-num-keys");
     } catch (RocksDBException e) {
       throw toIOException("Unable to get the estimated count.", e);
     }
@@ -244,7 +246,7 @@ public class RDBStore implements DBStore {
   @Override
   public ArrayList<Table> listTables() throws IOException {
     ArrayList<Table> returnList = new ArrayList<>();
-    for (ColumnFamilyHandle handle: handleTable.values())  {
+    for (ColumnFamilyHandle handle : handleTable.values()) {
       returnList.add(new RDBTable(db, handle, writeOptions));
     }
     return returnList;

hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TableConfig.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.utils.db;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyOptions;
+
+/**
+ * Class that maintains Table Configuration.
+ */
+public class TableConfig {
+  private final String name;
+  private final ColumnFamilyOptions columnFamilyOptions;
+
+
+  /**
+   * Constructs a Table Config.
+   * @param name - Name of the Table.
+   * @param columnFamilyOptions - Column Family options.
+   */
+  public TableConfig(String name, ColumnFamilyOptions columnFamilyOptions) {
+    this.name = name;
+    this.columnFamilyOptions = columnFamilyOptions;
+  }
+
+  /**
+   * Returns the Name for this Table.
+   * @return - Name String
+   */
+  public String getName() {
+    return name;
+  }
+
+  /**
+   * Returns a ColumnFamilyDescriptor for this table.
+   * @return ColumnFamilyDescriptor
+   */
+  public ColumnFamilyDescriptor getDescriptor() {
+    return  new ColumnFamilyDescriptor(DFSUtil.string2Bytes(name),
+        columnFamilyOptions);
+  }
+
+  /**
+   * Returns Column family options for this Table.
+   * @return  ColumnFamilyOptions used for the Table.
+   */
+  public ColumnFamilyOptions getColumnFamilyOptions() {
+    return columnFamilyOptions;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    TableConfig that = (TableConfig) o;
+    return new EqualsBuilder()
+        .append(getName(), that.getName())
+        .isEquals();
+  }
+
+  @Override
+  public int hashCode() {
+    return new HashCodeBuilder(17, 37)
+        .append(getName())
+        .toHashCode();
+  }
+}

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -1098,4 +1098,14 @@
       datanode unless the datanode confirms the completion.
     </description>
   </property>
+
+  <property>
+    <name>ozone.db.profile</name>
+    <value>DBProfile.SSD</value>
+    <tag>OZONE, OM, PERFORMANCE, REQUIRED</tag>
+    <description>This property allows user to pick a configuration
+    that tunes the RocksDB settings for the hardware it is running
+    on. Right now, we have SSD and DISK as profile options.</description>
+  </property>
+
 </configuration>

hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBConfigFromFile.java

@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.utils.db;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.DBOptions;
+import org.rocksdb.RocksDB;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.hadoop.utils.db.DBConfigFromFile.getOptionsFileNameFromDB;
+
+/**
+ * DBConf tests.
+ */
+public class TestDBConfigFromFile {
+  private final static String DB_FILE = "test.db";
+  private final static String INI_FILE = getOptionsFileNameFromDB(DB_FILE);
+  @Rule
+  public TemporaryFolder folder = new TemporaryFolder();
+
+  @Before
+  public void setUp() throws Exception {
+    System.setProperty(DBConfigFromFile.CONFIG_DIR,
+        folder.newFolder().toString());
+    ClassLoader classLoader = getClass().getClassLoader();
+    File testData = new File(classLoader.getResource(INI_FILE).getFile());
+    File dest = Paths.get(
+        System.getProperty(DBConfigFromFile.CONFIG_DIR), INI_FILE).toFile();
+    FileUtils.copyFile(testData, dest);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+  }
+
+  @Test
+  public void readFromFile() throws IOException {
+    final List<String> families =
+        Arrays.asList(DFSUtil.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY),
+            "First", "Second", "Third",
+            "Fourth", "Fifth",
+            "Sixth");
+    final List<ColumnFamilyDescriptor> columnFamilyDescriptors =
+        new ArrayList<>();
+    for (String family : families) {
+      columnFamilyDescriptors.add(
+          new ColumnFamilyDescriptor(family.getBytes(StandardCharsets.UTF_8),
+              new ColumnFamilyOptions()));
+    }
+
+    final DBOptions options = DBConfigFromFile.readFromFile(DB_FILE,
+        columnFamilyDescriptors);
+
+    // Some Random Values Defined in the test.db.ini, we verify that we are
+    // able to get values that are defined in the test.db.ini.
+    Assert.assertNotNull(options);
+    Assert.assertEquals(551615L, options.maxManifestFileSize());
+    Assert.assertEquals(1000L, options.keepLogFileNum());
+    Assert.assertEquals(1048576, options.writableFileMaxBufferSize());
+  }
+
+  @Test
+  public void readFromFileInvalidConfig() throws IOException {
+    final List<String> families =
+        Arrays.asList(DFSUtil.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY),
+            "First", "Second", "Third",
+            "Fourth", "Fifth",
+            "Sixth");
+    final List<ColumnFamilyDescriptor> columnFamilyDescriptors =
+        new ArrayList<>();
+    for (String family : families) {
+      columnFamilyDescriptors.add(
+          new ColumnFamilyDescriptor(family.getBytes(StandardCharsets.UTF_8),
+              new ColumnFamilyOptions()));
+    }
+
+    final DBOptions options = DBConfigFromFile.readFromFile("badfile.db.ini",
+        columnFamilyDescriptors);
+
+    // This has to return a Null, since we have config defined for badfile.db
+    Assert.assertNull(options);
+  }
+}

hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java

@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.utils.db;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+/**
+ * Tests RDBStore creation.
+ */
+public class TestDBStoreBuilder {
+
+  @Rule
+  public TemporaryFolder folder = new TemporaryFolder();
+  @Rule
+  public ExpectedException thrown = ExpectedException.none();
+
+  @Before
+  public void setUp() throws Exception {
+    System.setProperty(DBConfigFromFile.CONFIG_DIR,
+        folder.newFolder().toString());
+  }
+
+  @Test
+  public void builderWithoutAnyParams() throws IOException {
+    Configuration conf = new Configuration();
+    thrown.expect(IOException.class);
+    DBStoreBuilder.newBuilder(conf).build();
+  }
+
+  @Test
+  public void builderWithOneParamV1() throws IOException {
+    Configuration conf = new Configuration();
+    thrown.expect(IOException.class);
+    DBStoreBuilder.newBuilder(conf)
+        .setName("Test.db")
+        .build();
+  }
+
+  @Test
+  public void builderWithOneParamV2() throws IOException {
+    Configuration conf = new Configuration();
+    File newFolder = folder.newFolder();
+    if(!newFolder.exists()) {
+      Assert.assertTrue(newFolder.mkdirs());
+    }
+    thrown.expect(IOException.class);
+    DBStoreBuilder.newBuilder(conf)
+        .setPath(newFolder.toPath())
+        .build();
+  }
+
+  @Test
+  public void builderWithOpenClose() throws Exception {
+    Configuration conf = new Configuration();
+    File newFolder = folder.newFolder();
+    if(!newFolder.exists()) {
+      Assert.assertTrue(newFolder.mkdirs());
+    }
+    DBStore dbStore = DBStoreBuilder.newBuilder(conf)
+        .setName("Test.db")
+        .setPath(newFolder.toPath())
+        .build();
+    // Nothing to do just open and Close.
+    dbStore.close();
+  }
+
+  @Test
+  public void builderWithDoubleTableName() throws Exception {
+    Configuration conf = new Configuration();
+    File newFolder = folder.newFolder();
+    if(!newFolder.exists()) {
+      Assert.assertTrue(newFolder.mkdirs());
+    }
+    thrown.expect(IOException.class);
+    DBStoreBuilder.newBuilder(conf)
+        .setName("Test.db")
+        .setPath(newFolder.toPath())
+        .addTable("FIRST")
+        .addTable("FIRST")
+        .build();
+    // Nothing to do , This will throw so we do not have to close.
+
+  }
+
+  @Test
+  public void builderWithDataWrites() throws Exception {
+    Configuration conf = new Configuration();
+    File newFolder = folder.newFolder();
+    if(!newFolder.exists()) {
+      Assert.assertTrue(newFolder.mkdirs());
+    }
+    try (DBStore dbStore = DBStoreBuilder.newBuilder(conf)
+        .setName("Test.db")
+        .setPath(newFolder.toPath())
+        .addTable("First")
+        .addTable("Second")
+        .build()) {
+      try (Table firstTable = dbStore.getTable("First")) {
+        byte[] key =
+            RandomStringUtils.random(9).getBytes(StandardCharsets.UTF_8);
+        byte[] value =
+            RandomStringUtils.random(9).getBytes(StandardCharsets.UTF_8);
+        firstTable.put(key, value);
+        byte[] temp = firstTable.get(key);
+        Arrays.equals(value, temp);
+      }
+
+      try (Table secondTable = dbStore.getTable("Second")) {
+        Assert.assertTrue(secondTable.isEmpty());
+      }
+    }
+  }
+
+  @Test
+  public void builderWithDiskProfileWrites() throws Exception {
+    Configuration conf = new Configuration();
+    File newFolder = folder.newFolder();
+    if(!newFolder.exists()) {
+      Assert.assertTrue(newFolder.mkdirs());
+    }
+    try (DBStore dbStore = DBStoreBuilder.newBuilder(conf)
+        .setName("Test.db")
+        .setPath(newFolder.toPath())
+        .addTable("First")
+        .addTable("Second")
+        .setProfile(DBProfile.DISK)
+        .build()) {
+      try (Table firstTable = dbStore.getTable("First")) {
+        byte[] key =
+            RandomStringUtils.random(9).getBytes(StandardCharsets.UTF_8);
+        byte[] value =
+            RandomStringUtils.random(9).getBytes(StandardCharsets.UTF_8);
+        firstTable.put(key, value);
+        byte[] temp = firstTable.get(key);
+        Arrays.equals(value, temp);
+      }
+
+      try (Table secondTable = dbStore.getTable("Second")) {
+        Assert.assertTrue(secondTable.isEmpty());
+      }
+    }
+  }
+
+
+}

hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java

@@ -28,6 +28,7 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.junit.rules.TemporaryFolder;
+import org.rocksdb.ColumnFamilyOptions;
 import org.rocksdb.DBOptions;
 import org.rocksdb.RocksDB;
 import org.rocksdb.Statistics;
@@ -39,8 +40,10 @@ import java.lang.management.ManagementFactory;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * RDBStore Tests.
@@ -57,6 +60,7 @@ public class TestRDBStore {
   public ExpectedException thrown = ExpectedException.none();
   private RDBStore rdbStore = null;
   private DBOptions options = null;
+  private Set<TableConfig> configSet;
 
   @Before
   public void setUp() throws Exception {
@@ -67,7 +71,12 @@ public class TestRDBStore {
     Statistics statistics = new Statistics();
     statistics.setStatsLevel(StatsLevel.ALL);
     options = options.setStatistics(statistics);
-    rdbStore = new RDBStore(folder.newFolder(), options, families);
+    configSet = new HashSet<>();
+    for(String name : families) {
+      TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
+      configSet.add(newConfig);
+    }
+    rdbStore = new RDBStore(folder.newFolder(), options, configSet);
   }
 
   @After
@@ -80,7 +89,7 @@ public class TestRDBStore {
   @Test
   public void compactDB() throws Exception {
     try (RDBStore newStore =
-             new RDBStore(folder.newFolder(), options, families)) {
+             new RDBStore(folder.newFolder(), options, configSet)) {
       Assert.assertNotNull("DB Store cannot be null", newStore);
       try (Table firstTable = newStore.getTable(families.get(1))) {
         Assert.assertNotNull("Table cannot be null", firstTable);
@@ -101,7 +110,7 @@ public class TestRDBStore {
   @Test
   public void close() throws Exception {
     RDBStore newStore =
-        new RDBStore(folder.newFolder(), options, families);
+        new RDBStore(folder.newFolder(), options, configSet);
     Assert.assertNotNull("DBStore cannot be null", newStore);
     // This test does not assert anything if there is any error this test
     // will throw and fail.
@@ -157,7 +166,7 @@ public class TestRDBStore {
   @Test
   public void getEstimatedKeyCount() throws Exception {
     try (RDBStore newStore =
-             new RDBStore(folder.newFolder(), options, families)) {
+             new RDBStore(folder.newFolder(), options, configSet)) {
       Assert.assertNotNull("DB Store cannot be null", newStore);
       // Write 100 keys to the first table.
       try (Table firstTable = newStore.getTable(families.get(1))) {

hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBTableStore.java

@@ -27,6 +27,7 @@ import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.rocksdb.ColumnFamilyOptions;
 import org.rocksdb.DBOptions;
 import org.rocksdb.RocksDB;
 import org.rocksdb.Statistics;
@@ -35,8 +36,10 @@ import org.rocksdb.WriteBatch;
 
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Set;
 
 /**
  * Tests for RocksDBTable Store.
@@ -62,7 +65,13 @@ public class TestRDBTableStore {
     Statistics statistics = new Statistics();
     statistics.setStatsLevel(StatsLevel.ALL);
     options = options.setStatistics(statistics);
-    rdbStore = new RDBStore(folder.newFolder(), options, families);
+
+    Set<TableConfig> configSet = new HashSet<>();
+    for(String name : families) {
+      TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
+      configSet.add(newConfig);
+    }
+    rdbStore = new RDBStore(folder.newFolder(), options, configSet);
   }
 
   @After

hadoop-hdds/common/src/test/resources/test.db.ini

@@ -0,0 +1,145 @@
+# This is a RocksDB option file.
+#
+# A typical RocksDB options file has four sections, which are
+# Version section, DBOptions section, at least one CFOptions
+# section, and one TableOptions section for each column family.
+# The RocksDB options file in general follows the basic INI
+# file format with the following extensions / modifications:
+#
+#  * Escaped characters
+#    We escaped the following characters:
+#     - \n -- line feed - new line
+#     - \r -- carriage return
+#     - \\ -- backslash \
+#     - \: -- colon symbol :
+#     - \# -- hash tag #
+#  * Comments
+#    We support # style comments.  Comments can appear at the ending
+#    part of a line.
+#  * Statements
+#    A statement is of the form option_name = value.
+#    Each statement contains a '=', where extra white-spaces
+#    are supported. However, we don't support multi-lined statement.
+#    Furthermore, each line can only contain at most one statement.
+#  * Sections
+#    Sections are of the form [SecitonTitle "SectionArgument"],
+#    where section argument is optional.
+#  * List
+#    We use colon-separated string to represent a list.
+#    For instance, n1:n2:n3:n4 is a list containing four values.
+#
+# Below is an example of a RocksDB options file:
+
+
+#----------------------IMPORTANT------------------------------------#
+### FAKE VALUES FOR TESTING ONLY ### DO NOT USE THESE FOR PRODUCTION.
+#----------------------IMPORTANT------------------------------------#
+[DBOptions]
+  stats_dump_period_sec=600
+  max_manifest_file_size=551615
+  bytes_per_sync=8388608
+  delayed_write_rate=2097152
+  WAL_ttl_seconds=0
+  WAL_size_limit_MB=0
+  max_subcompactions=1
+  wal_dir=
+  wal_bytes_per_sync=0
+  db_write_buffer_size=0
+  keep_log_file_num=1000
+  table_cache_numshardbits=4
+  max_file_opening_threads=1
+  writable_file_max_buffer_size=1048576
+  random_access_max_buffer_size=1048576
+  use_fsync=false
+  max_total_wal_size=0
+  max_open_files=-1
+  skip_stats_update_on_db_open=false
+  max_background_compactions=16
+  manifest_preallocation_size=4194304
+  max_background_flushes=7
+  is_fd_close_on_exec=true
+  max_log_file_size=0
+  advise_random_on_open=true
+  create_missing_column_families=false
+  paranoid_checks=true
+  delete_obsolete_files_period_micros=21600000000
+  log_file_time_to_roll=0
+  compaction_readahead_size=0
+  create_if_missing=false
+  use_adaptive_mutex=false
+  enable_thread_tracking=false
+  allow_fallocate=true
+  error_if_exists=false
+  recycle_log_file_num=0
+  skip_log_error_on_recovery=false
+  db_log_dir=
+  new_table_reader_for_compaction_inputs=true
+  allow_mmap_reads=false
+  allow_mmap_writes=false
+  use_direct_reads=false
+  use_direct_writes=false
+
+
+[CFOptions "default"]
+  compaction_style=kCompactionStyleLevel
+  compaction_filter=nullptr
+  num_levels=6
+  table_factory=BlockBasedTable
+  comparator=leveldb.BytewiseComparator
+  max_sequential_skip_in_iterations=8
+  soft_rate_limit=0.000000
+  max_bytes_for_level_base=1073741824
+  memtable_prefix_bloom_probes=6
+  memtable_prefix_bloom_bits=0
+  memtable_prefix_bloom_huge_page_tlb_size=0
+  max_successive_merges=0
+  arena_block_size=16777216
+  min_write_buffer_number_to_merge=1
+  target_file_size_multiplier=1
+  source_compaction_factor=1
+  max_bytes_for_level_multiplier=8
+  max_bytes_for_level_multiplier_additional=2:3:5
+  compaction_filter_factory=nullptr
+  max_write_buffer_number=8
+  level0_stop_writes_trigger=20
+  compression=kSnappyCompression
+  level0_file_num_compaction_trigger=4
+  purge_redundant_kvs_while_flush=true
+  max_write_buffer_number_to_maintain=0
+  memtable_factory=SkipListFactory
+  max_grandparent_overlap_factor=8
+  expanded_compaction_factor=25
+  hard_pending_compaction_bytes_limit=137438953472
+  inplace_update_num_locks=10000
+  level_compaction_dynamic_level_bytes=true
+  level0_slowdown_writes_trigger=12
+  filter_deletes=false
+  verify_checksums_in_compaction=true
+  min_partial_merge_operands=2
+  paranoid_file_checks=false
+  target_file_size_base=134217728
+  optimize_filters_for_hits=false
+  merge_operator=PutOperator
+  compression_per_level=kNoCompression:kNoCompression:kNoCompression:kSnappyCompression:kSnappyCompression:kSnappyCompression
+  compaction_measure_io_stats=false
+  prefix_extractor=nullptr
+  bloom_locality=0
+  write_buffer_size=134217728
+  disable_auto_compactions=false
+  inplace_update_support=false
+
+[TableOptions/BlockBasedTable "default"]
+  format_version=2
+  whole_key_filtering=true
+  no_block_cache=false
+  checksum=kCRC32c
+  filter_policy=rocksdb.BuiltinBloomFilter
+  block_size_deviation=10
+  block_size=8192
+  block_restart_interval=16
+  cache_index_and_filter_blocks=false
+  pin_l0_filter_and_index_blocks_in_cache=false
+  pin_top_level_index_and_filter=false
+  index_type=kBinarySearch
+  hash_index_allow_collision=true
+  flush_block_policy_factory=FlushBlockBySizePolicyFactory

hadoop-hdds/pom.xml

@@ -105,6 +105,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
             <exclude>src/test/resources/incorrect.container</exclude>
             <exclude>src/test/resources/additionalfields.container</exclude>
             <exclude>src/test/resources/incorrect.checksum.container</exclude>
+            <exclude>src/test/resources/test.db.ini</exclude>
           </excludes>
         </configuration>
       </plugin>