diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml index 0997f2a5e7d..5e5469ba466 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml @@ -26,6 +26,7 @@ ozone.enabled false + OZONE,REQUIRED Status of the Ozone Object Storage service is enabled. Set to true to enable Ozone. @@ -36,6 +37,7 @@ ozone.handler.type distributed + OZONE,REST The second key dfs.storage.handler.type tells ozone which storage handler to use. The possible values are: @@ -47,6 +49,7 @@ ozone.localstorage.root /tmp/ozone + OZONE,REST @@ -54,6 +57,7 @@ ozone.trace.enabled false + OZONE,DEBUG @@ -61,6 +65,7 @@ ozone.metadata.dirs + OZONE,KSM,SCM,CONTAINER,REQUIRED,STORAGE Ozone metadata dir path. @@ -69,6 +74,7 @@ ozone.key.cache.size 1024 + PERFORMANCE,CONTAINER,STORAGE @@ -76,6 +82,7 @@ ozone.administrators + OZONE,SECURITY Ozone administrator users delimited by comma. If not set, only the user who launches an ozone service will be the @@ -88,6 +95,7 @@ ozone.client.protocol org.apache.hadoop.ozone.client.rpc.RpcClient + OZONE Protocol class to be used by client to connect to ozone cluster. The build-in implementation includes: @@ -99,6 +107,7 @@ ozone.client.socket.timeout.ms 5000 + OZONE Socket timeout for Ozone client in milliseconds. @@ -107,6 +116,7 @@ ozone.client.connection.timeout.ms 5000 + OZONE,PERFORMANCE,CLIENT Connection timeout for Ozone client in milliseconds. @@ -115,6 +125,7 @@ ozone.scm.client.address + OZONE The address of the Ozone SCM client service. This is a required setting. @@ -127,6 +138,7 @@ ozone.scm.client.port 9860 + OZONE The port number of the Ozone SCM client service. @@ -135,6 +147,7 @@ ozone.scm.datanode.address + OZONE The address of the Ozone SCM service used for internal communication between the DataNodes and the SCM. @@ -151,6 +164,7 @@ ozone.scm.datanode.bind.host + OZONE The hostname or IP address used by the SCM service endpoint to bind. @@ -159,6 +173,7 @@ ozone.scm.datanode.port 9861 + OZONE The port number of the Ozone SCM service. @@ -167,6 +182,7 @@ ozone.scm.client.bind.host 0.0.0.0 + OZONE The hostname or IP address used by the SCM client endpoint to bind. This setting is used by the SCM only and never used by clients. @@ -181,6 +197,7 @@ ozone.scm.block.client.address + OZONE The address of the Ozone SCM block client service. If not defined value of ozone.scm.client.address is used. @@ -190,6 +207,7 @@ ozone.scm.block.client.bind.host 0.0.0.0 + OZONE The hostname or IP address used by the SCM block client endpoint to bind. @@ -199,6 +217,7 @@ ozone.scm.block.client.port 9863 + OZONE The port number of the Ozone SCM block client service. @@ -207,6 +226,7 @@ ozone.scm.handler.count.key 10 + OZONE The number of RPC handler threads for each SCM service endpoint. @@ -217,6 +237,7 @@ ozone.scm.chunk.size 1048576 + OZONE The chunk size for read/write chunk operations in bytes. @@ -229,6 +250,7 @@ ozone.scm.heartbeat.interval.seconds 30 + OZONE The heartbeat interval from a datanode to SCM in seconds. @@ -237,6 +259,7 @@ ozone.scm.stale.node.interval.ms 90000 + OZONE The interval in milliseconds for stale node flagging. @@ -245,6 +268,7 @@ ozone.scm.dead.node.interval.ms 600000 + OZONE The interval in milliseconds for dead node flagging. @@ -253,6 +277,7 @@ ozone.scm.max.hb.count.to.process 5000 + OZONE The maximum number of heartbeat to process per loop of the process thread. @@ -262,6 +287,7 @@ ozone.scm.heartbeat.thread.interval.ms 3000 + OZONE The interval in milliseconds that the heartbeat processor thread runs. @@ -270,6 +296,7 @@ ozone.scm.heartbeat.rpc-timeout 1000 + OZONE Timeout value for the RPC from Datanode to SCM in milliseconds. @@ -278,6 +305,7 @@ ozone.scm.block.deletion.max.retry 4096 + OZONE SCM wraps up a number of blocks in a deletion transaction and send that to datanode for physically deletion periodically. This property @@ -291,6 +319,7 @@ ozone.scm.heartbeat.log.warn.interval.count 10 + OZONE Defines how frequently we will log the missing of heartbeat to SCM. For example in the default case, we will write a warning message for each 10 @@ -301,6 +330,7 @@ ozone.scm.names + OZONE The value of this property is a set of DNS | DNS:PORT | IP Address | IP:PORT. Written as a comma separated string. e.g. scm1, scm2:8020, 7.7.7.7:7777. @@ -312,6 +342,7 @@ ozone.scm.datanode.id + OZONE The path that datanodes will created to store the datanode ID. If this value is not set, datanodes will fail to come up. @@ -321,6 +352,7 @@ ozone.scm.db.cache.size.mb 128 + OZONE The cache size for SCM level db store in MB. @@ -329,6 +361,7 @@ ozone.scm.container.size.gb 5 + OZONE Storage container size in GB for block allocations. @@ -337,6 +370,7 @@ ozone.scm.container.placement.impl org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementRandom + OZONE Placement policy class for containers. Defaults to SCMContainerPlacementRandom.class @@ -346,6 +380,7 @@ ozone.scm.container.provision_batch_size 1 + OZONE Pre-provision specified number of containers to creare for block creation. @@ -355,6 +390,7 @@ ozone.scm.max.container.report.threads 100 + OZONE Maximum number of threads to process container reports in scm. Each container report from a datanode is processed by scm in @@ -366,6 +402,7 @@ ozone.scm.container.reports.wait.timeout.seconds 300 + OZONE Maximum time to wait in seconds for processing all container reports from a node pool. It determines the timeout for a @@ -376,6 +413,7 @@ ozone.scm.container.report.processing.interval.seconds 60 + OZONE Time interval in seconds for scm to process container reports for a node pool. Scm handles node pool reports in a cyclic clock @@ -386,6 +424,7 @@ ozone.block.deleting.service.interval.ms 60000 + OZONE,PERFORMANCE,SCM Time interval in milliseconds of the block deleting service. The block deleting service runs on each datanode to scan staled @@ -396,6 +435,7 @@ ozone.block.deleting.service.timeout 300000ms + OZONE,PERFORMANCE,SCM Timeout value of block deletion service. If this is set greater than 0, the service will stop waiting for the block deleting completion after this @@ -409,6 +449,7 @@ ozone.block.deleting.limit.per.task 1000 + OZONE,PERFORMANCE,SCM Maximum number of blocks to be deleted by block deleting service per time interval. This property is used to throttle the actual number @@ -419,6 +460,7 @@ ozone.block.deleting.container.limit.per.interval 10 + OZONE,PERFORMANCE,SCM Maximum number of containers to be scanned by block deleting service per time interval. The block deleting service spawns a thread to handle @@ -430,6 +472,7 @@ ozone.scm.container.deletion-choosing.policy org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy + OZONE The policy used for choosing desire containers for block deletion. Datanode selects a number of containers to process block deletion @@ -450,6 +493,7 @@ ozone.key.deleting.limit.per.task 1000 + PERFORMANCE,KSM Maximum number of keys to be scanned by key deleting service per time interval in KSM. Those keys are sent to delete metadata and @@ -461,6 +505,7 @@ dfs.container.ipc 9859 + OZONE The ipc port number of container. @@ -469,6 +514,7 @@ dfs.container.ipc.random.port false + OZONE,CONTAINER,PIPELINE,STANDALONE Whether allocates a random free port for ozone container. @@ -477,6 +523,7 @@ scm.container.client.idle.threshold 10000 + OZONE @@ -484,6 +531,7 @@ scm.container.client.max.size 256 + OZONE @@ -491,6 +539,7 @@ dfs.container.ratis.enabled false + OZONE @@ -498,6 +547,7 @@ dfs.container.ratis.rpc.type GRPC + OZONE @@ -505,6 +555,7 @@ dfs.container.ratis.server.id + OZONE The unique ID to identify a Ratis server. @@ -513,6 +564,7 @@ dfs.container.ratis.datanode.storage.dir + OZONE,CONTAINER,STORAGE @@ -522,6 +574,7 @@ ozone.ksm.handler.count.key 200 + PERFORMANCE,KSM The number of RPC handler threads for each KSM service endpoint. @@ -530,6 +583,7 @@ ozone.ksm.address + KSM,REQUIRED The address of the Ozone KSM service. @@ -538,6 +592,7 @@ ozone.ksm.leveldb.cache.size.mb 128 + PERFORMANCE,KSM The size of KSM LevelDB cache in MB that used for caching files. @@ -546,6 +601,7 @@ ozone.ksm.user.max.volume 1024 + KSM,MANAGEMENT The maximum number of volumes that each user can create. @@ -554,6 +610,7 @@ ozone.ksm.user.rights READ_WRITE + SECURITY,KSM Default user permissions in Ozone KSM. @@ -562,6 +619,7 @@ ozone.ksm.group.rights READ_WRITE + SECURITY,KSM Default group permissions in Ozone KSM. @@ -571,6 +629,7 @@ ozone.scm.http.enabled true + OZONE Property to enable or disable SCM web ui. @@ -579,6 +638,7 @@ ozone.scm.http-address 0.0.0.0:9876 + OZONE The address and the base port where the SCM web ui will listen on. @@ -589,6 +649,7 @@ ozone.scm.http-bind-host 0.0.0.0 + OZONE The actual address the SCM web server will bind to. If this optional address is set, it overrides only the hostname portion of @@ -599,6 +660,7 @@ ozone.scm.https-address 0.0.0.0:9877 + OZONE The address and the base port where the SCM web ui will listen on using HTTPS. @@ -610,6 +672,7 @@ ozone.scm.https-bind-host 0.0.0.0 + OZONE The actual address the SCM web server will bind to using HTTPS. If this optional address is set, it overrides only the hostname portion of @@ -620,6 +683,7 @@ ozone.ksm.http.enabled true + KSM,MANAGEMENT Property to enable or disable KSM web ui. @@ -628,6 +692,7 @@ ozone.ksm.http-address 0.0.0.0:9874 + KSM,MANAGEMENT The address and the base port where the KSM web ui will listen on. @@ -638,6 +703,7 @@ ozone.ksm.http-bind-host 0.0.0.0 + KSM,MANAGEMENT The actual address the KSM web server will bind to. If this optional address is set, it overrides only the hostname portion of @@ -648,6 +714,7 @@ ozone.ksm.https-address 0.0.0.0:9875 + KSM,MANAGEMENT,SECURITY The address and the base port where the KSM web ui will listen on using HTTPS. @@ -659,6 +726,7 @@ ozone.ksm.https-bind-host 0.0.0.0 + KSM,MANAGEMENT,SECURITY The actual address the KSM web server will bind to using HTTPS. If this optional address is set, it overrides only the hostname portion of @@ -669,6 +737,7 @@ ozone.web.authentication.kerberos.principal + OZONE,SECURITY The server principal used by the SCM and KSM for web UI SPNEGO authentication when Kerberos security is enabled. This is @@ -683,6 +752,7 @@ ozone.scm.keytab.file + OZONE The keytab file for Kerberos authentication in SCM. @@ -691,6 +761,7 @@ ozone.ksm.keytab.file + SECURITY,KSM The keytab file for Kerberos authentication in KSM. @@ -699,6 +770,7 @@ ozone.metastore.impl RocksDB + OZONE,KSM,SCM,CONTAINER,STORAGE Ozone metadata store implementation. Ozone metadata are well distributed to multiple services such as ksm, scm. They are stored in some local @@ -710,6 +782,7 @@ dfs.cblock.servicerpc-address + OZONE The address that cblock will be bind to, should be a host:port format, this setting is required for cblock server to start. @@ -721,6 +794,7 @@ dfs.cblock.service.rpc-bind-host 0.0.0.0 + OZONE The actual address the cblock service rpc server will bind to. If this optional address is set, it overrides only the hostname portion of @@ -731,6 +805,7 @@ dfs.cblock.jscsi-address + OZONE The address that cblock will be bind to, should be a host:port format, this setting is required for cblock server to start. @@ -741,6 +816,7 @@ dfs.cblock.jscsi.rpc-bind-host 0.0.0.0 + OZONE The actual address the cblock jscsi rpc server will bind to. If this optional address is set, it overrides only the hostname portion of @@ -751,6 +827,7 @@ dfs.cblock.jscsi.port 9811 + OZONE The port on CBlockManager node for jSCSI to talk to. @@ -759,6 +836,7 @@ dfs.storage.service.handler.count 10 + OZONE Default number of handlers for CBlock service rpc. @@ -767,6 +845,7 @@ dfs.cblock.service.leveldb.path /tmp/cblock_levelDB.dat + OZONE Default path for the cblock meta data disk store. @@ -775,6 +854,7 @@ dfs.cblock.disk.cache.path /tmp/cblockCacheDB + OZONE Default path for the cblock local cache. @@ -783,6 +863,7 @@ dfs.cblock.trace.io false + OZONE Default flag for enabling trace io. @@ -791,6 +872,7 @@ dfs.cblock.short.circuit.io false + OZONE Default flag to enable cblock local cache. @@ -799,6 +881,7 @@ dfs.cblock.cache.cache.size.in.kb 256 + OZONE Default cblock cache queue size, in number of kb. @@ -807,6 +890,7 @@ dfs.cblock.cache.core.min.pool.size 16 + OZONE Minimum number of thread pool thread that cBlock cache will use for background I/O. @@ -815,6 +899,7 @@ dfs.cblock.cache.max.pool.size 256 + OZONE Maximum number of thread pool thread that cBlcok cache will use for background I/O. @@ -823,6 +908,7 @@ dfs.cblock.cache.keep.alive.seconds 60 + OZONE Number of seconds to keep the Thread alive when it is idle. @@ -831,6 +917,7 @@ dfs.cblock.cache.thread.priority 5 + OZONE Priority of cache flusher thread, affecting the relative performance of write and read.upported values are 1, 5, 10. @@ -841,6 +928,7 @@ dfs.cblock.cache.block.buffer.size 512 + OZONE Block Buffer size in terms of blockID entries, in number of blockIDs. @@ -849,6 +937,7 @@ dfs.cblock.block.buffer.flush.interval.seconds 60 + OZONE The interval to flush cblock buffer in seconds. @@ -857,6 +946,7 @@ dfs.cblock.jscsi.server.address 0.0.0.0 + OZONE The address that jscsi server will be running, should have one local jscsi server for each client that tries to mount cblock. @@ -866,6 +956,7 @@ dfs.cblock.jscsi.cblock.server.address 127.0.0.1 + OZONE The address local jscsi server will use to talk to cblock manager. @@ -874,6 +965,7 @@ dfs.cblock.container.size 5 + OZONE The size of ozone container in number of GBs. Note that this is not setting container size for ozone, but rather, this is setting @@ -884,6 +976,7 @@ dfs.cblock.cache.leveldb.cache.size.mb 256 + OZONE The size of LevelDB cache file which uses an off-heap cache in LevelDB. @@ -892,6 +985,7 @@ dfs.cblock.cache.max.retry 65536 + OZONE The maximum number of retries when writing a block to container. @@ -900,6 +994,7 @@ dfs.cblock.manager.pool.size 16 + OZONE Number of threads that cblock manager will use for container operations. @@ -908,6 +1003,7 @@ dfs.cblock.rpc.timeout.seconds 300 + OZONE RPC timeout in seconds used for cblock CLI operations. @@ -916,6 +1012,7 @@ dfs.cblock.scm.ipaddress 127.0.0.1 + OZONE IP address used by cblock to connect to SCM. @@ -924,6 +1021,7 @@ dfs.cblock.scm.port 9860 + OZONE Port used by cblock to connect to SCM. @@ -932,6 +1030,7 @@ ozone.scm.block.size 268435456 + OZONE The default size of a scm block in bytes. @@ -940,6 +1039,7 @@ dfs.container.ratis.ipc 9858 + OZONE,CONTAINER,PIPELINE,RATIS The ipc port number of container. @@ -948,6 +1048,7 @@ dfs.container.ratis.ipc.random.port false + OZONE,CONTAINER,PIPELINE,RATIS Whether allocates a random free port for ozone ratis port for container. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java index d8b7595a08a..b4f4fee937b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java @@ -18,9 +18,13 @@ package org.apache.hadoop.ozone; import org.apache.hadoop.cblock.CBlockConfigKeys; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.OzonePropertyTag; import org.apache.hadoop.conf.TestConfigurationFieldsBase; import org.apache.hadoop.ozone.ksm.KSMConfigKeys; import org.apache.hadoop.scm.ScmConfigKeys; +import org.junit.Test; +import static org.junit.Assert.assertEquals; /** * Tests if configuration constants documented in ozone-defaults.xml. @@ -37,4 +41,31 @@ public class TestOzoneConfigurationFields extends TestConfigurationFieldsBase { errorIfMissingXmlProps = true; } + @Test + public void testOzoneTags() { + Configuration config = new OzoneConfiguration(); + config.reloadConfiguration(); + + // To load default resources + config.get("ozone.enabled"); + assertEquals(87, + config.getAllPropertiesByTag(OzonePropertyTag.OZONE).size()); + assertEquals(15, config.getAllPropertiesByTag(OzonePropertyTag.KSM) + .size()); + assertEquals(6, config.getAllPropertiesByTag(OzonePropertyTag.SCM) + .size()); + assertEquals(6, config.getAllPropertiesByTag(OzonePropertyTag.MANAGEMENT) + .size()); + assertEquals(7, config.getAllPropertiesByTag(OzonePropertyTag.SECURITY) + .size()); + assertEquals(9, config.getAllPropertiesByTag(OzonePropertyTag.PERFORMANCE) + .size()); + assertEquals(1, config.getAllPropertiesByTag(OzonePropertyTag.DEBUG) + .size()); + assertEquals(3, config.getAllPropertiesByTag(OzonePropertyTag.REQUIRED) + .size()); + assertEquals(2, config.getAllPropertiesByTag(OzonePropertyTag.RATIS) + .size()); + } + } \ No newline at end of file