HADOOP-16656. Document FairCallQueue configs in core-default.xml. Contributed by Siyao Meng.
Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
This commit is contained in:
Siyao Meng
Wei-Chiu Chuang
parent
40150da1e1
commit
3ab98eefa5
|
|
@ -2273,6 +2273,193 @@
|
|||
because the server side is stuck in TIME_WAIT state.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<!-- FairCallQueue properties -->
|
||||
<!-- See FairCallQueue documentation for a table of all properties -->
|
||||
|
||||
<!-- [port_number] is the port used by the IPC server to be configured. -->
|
||||
<!-- For example, ipc.8020.callqueue.impl will adjust the call queue -->
|
||||
<!-- implementation for the IPC server running at port 8020. -->
|
||||
|
||||
<!-- Typically, [port_number] is configured to be the NameNode RPC port, -->
|
||||
<!-- i.e. port number in dfs.namenode.rpc-address, or port number in -->
|
||||
<!-- fs.defaultFS if dfs.namenode.rpc-address is not explicitly configured. -->
|
||||
<property>
|
||||
<name>ipc.[port_number].backoff.enable</name>
|
||||
<value>false</value>
|
||||
<description>Whether or not to enable client backoff when a queue is full.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].callqueue.impl</name>
|
||||
<value>java.util.concurrent.LinkedBlockingQueue</value>
|
||||
<description>The fully qualified name of a class to use as the implementation
|
||||
of a call queue. The default implementation is
|
||||
java.util.concurrent.LinkedBlockingQueue (FIFO queue).
|
||||
Use org.apache.hadoop.ipc.FairCallQueue for the Fair Call Queue.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].scheduler.impl</name>
|
||||
<value>org.apache.hadoop.ipc.DefaultRpcScheduler</value>
|
||||
<description>The fully qualified name of a class to use as the
|
||||
implementation of the scheduler. The default implementation is
|
||||
org.apache.hadoop.ipc.DefaultRpcScheduler (no-op scheduler) when not using
|
||||
FairCallQueue. If using FairCallQueue, defaults to
|
||||
org.apache.hadoop.ipc.DecayRpcScheduler. Use
|
||||
org.apache.hadoop.ipc.DecayRpcScheduler in conjunction with the Fair Call
|
||||
Queue.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].scheduler.priority.levels</name>
|
||||
<value>4</value>
|
||||
<description>How many priority levels to use within the scheduler and call
|
||||
queue. This property applies to RpcScheduler and CallQueue.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].faircallqueue.multiplexer.weights</name>
|
||||
<value>8,4,2,1</value>
|
||||
<description>How much weight to give to each priority queue. This should be
|
||||
a comma-separated list of length equal to the number of priority levels.
|
||||
Weights descend by a factor of 2 (e.g., for 4 levels: 8,4,2,1).
|
||||
This property applies to WeightedRoundRobinMultiplexer.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].identity-provider.impl</name>
|
||||
<value>org.apache.hadoop.ipc.UserIdentityProvider</value>
|
||||
<description>The identity provider mapping user requests to their identity.
|
||||
This property applies to DecayRpcScheduler.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].cost-provider.impl</name>
|
||||
<value>org.apache.hadoop.ipc.DefaultCostProvider</value>
|
||||
<description>The cost provider mapping user requests to their cost. To
|
||||
enable determination of cost based on processing time, use
|
||||
org.apache.hadoop.ipc.WeightedTimeCostProvider.
|
||||
This property applies to DecayRpcScheduler.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].decay-scheduler.period-ms</name>
|
||||
<value>5000</value>
|
||||
<description>How frequently the decay factor should be applied to the
|
||||
operation counts of users. Higher values have less overhead, but respond
|
||||
less quickly to changes in client behavior.
|
||||
This property applies to DecayRpcScheduler.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].decay-scheduler.decay-factor</name>
|
||||
<value>0.5</value>
|
||||
<description>When decaying the operation counts of users, the multiplicative
|
||||
decay factor to apply. Higher values will weight older operations more
|
||||
strongly, essentially giving the scheduler a longer memory, and penalizing
|
||||
heavy clients for a longer period of time.
|
||||
This property applies to DecayRpcScheduler.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].decay-scheduler.thresholds</name>
|
||||
<value>13,25,50</value>
|
||||
<description>The client load threshold, as an integer percentage, for each
|
||||
priority queue. Clients producing less load, as a percent of total
|
||||
operations, than specified at position i will be given priority i. This
|
||||
should be a comma-separated list of length equal to the number of priority
|
||||
levels minus 1 (the last is implicitly 100).
|
||||
Thresholds ascend by a factor of 2 (e.g., for 4 levels: 13,25,50).
|
||||
This property applies to DecayRpcScheduler.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].decay-scheduler.backoff.responsetime.enable</name>
|
||||
<value>false</value>
|
||||
<description>Whether or not to enable the backoff by response time feature.
|
||||
This property applies to DecayRpcScheduler.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds</name>
|
||||
<value>10s,20s,30s,40s</value>
|
||||
<description>The response time thresholds, as time durations, for each
|
||||
priority queue. If the average response time for a queue is above this
|
||||
threshold, backoff will occur in lower priority queues. This should be a
|
||||
comma-separated list of length equal to the number of priority levels.
|
||||
Threshold increases by 10s per level (e.g., for 4 levels: 10s,20s,30s,40s)
|
||||
This property applies to DecayRpcScheduler.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].decay-scheduler.metrics.top.user.count</name>
|
||||
<value>10</value>
|
||||
<description>The number of top (i.e., heaviest) users to emit metric
|
||||
information about. This property applies to DecayRpcScheduler.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].weighted-cost.lockshared</name>
|
||||
<value>10</value>
|
||||
<description>The weight multiplier to apply to the time spent in the
|
||||
processing phase which holds a shared (read) lock.
|
||||
This property applies to WeightedTimeCostProvider.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].weighted-cost.lockexclusive</name>
|
||||
<value>100</value>
|
||||
<description>The weight multiplier to apply to the time spent in the
|
||||
processing phase which holds an exclusive (write) lock.
|
||||
This property applies to WeightedTimeCostProvider.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].weighted-cost.handler</name>
|
||||
<value>1</value>
|
||||
<description>The weight multiplier to apply to the time spent in the
|
||||
HANDLER phase which do not involve holding a lock.
|
||||
See org.apache.hadoop.ipc.ProcessingDetails.Timing for more details on
|
||||
this phase. This property applies to WeightedTimeCostProvider.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].weighted-cost.lockfree</name>
|
||||
<value>1</value>
|
||||
<description>The weight multiplier to apply to the time spent in the
|
||||
LOCKFREE phase which do not involve holding a lock.
|
||||
See org.apache.hadoop.ipc.ProcessingDetails.Timing for more details on
|
||||
this phase. This property applies to WeightedTimeCostProvider.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>ipc.[port_number].weighted-cost.response</name>
|
||||
<value>1</value>
|
||||
<description>The weight multiplier to apply to the time spent in the
|
||||
RESPONSE phase which do not involve holding a lock.
|
||||
See org.apache.hadoop.ipc.ProcessingDetails.Timing for more details on
|
||||
this phase. This property applies to WeightedTimeCostProvider.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<!-- Proxy Configuration -->
|
||||
|
||||
<property>
|
||||
|
|
|
|||
|
|
@ -135,6 +135,30 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
|
|||
xmlPropsToSkipCompare.add("fs.azure.saskey.usecontainersaskeyforallaccess");
|
||||
xmlPropsToSkipCompare.add("fs.azure.user.agent.prefix");
|
||||
|
||||
// FairCallQueue configs that includes dynamic ports in its keys
|
||||
xmlPropsToSkipCompare.add("ipc.[port_number].backoff.enable");
|
||||
xmlPropsToSkipCompare.add("ipc.[port_number].callqueue.impl");
|
||||
xmlPropsToSkipCompare.add("ipc.[port_number].scheduler.impl");
|
||||
xmlPropsToSkipCompare.add("ipc.[port_number].scheduler.priority.levels");
|
||||
xmlPropsToSkipCompare.add(
|
||||
"ipc.[port_number].faircallqueue.multiplexer.weights");
|
||||
xmlPropsToSkipCompare.add("ipc.[port_number].identity-provider.impl");
|
||||
xmlPropsToSkipCompare.add("ipc.[port_number].cost-provider.impl");
|
||||
xmlPropsToSkipCompare.add("ipc.[port_number].decay-scheduler.period-ms");
|
||||
xmlPropsToSkipCompare.add("ipc.[port_number].decay-scheduler.decay-factor");
|
||||
xmlPropsToSkipCompare.add("ipc.[port_number].decay-scheduler.thresholds");
|
||||
xmlPropsToSkipCompare.add(
|
||||
"ipc.[port_number].decay-scheduler.backoff.responsetime.enable");
|
||||
xmlPropsToSkipCompare.add(
|
||||
"ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds");
|
||||
xmlPropsToSkipCompare.add(
|
||||
"ipc.[port_number].decay-scheduler.metrics.top.user.count");
|
||||
xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.lockshared");
|
||||
xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.lockexclusive");
|
||||
xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.handler");
|
||||
xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.lockfree");
|
||||
xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.response");
|
||||
|
||||
// Deprecated properties. These should eventually be removed from the
|
||||
// class.
|
||||
configurationPropsToSkipCompare
|
||||
|
|
|
|||
Loading…
Reference in New Issue