YARN-10412. Move CS placement rule related changes to a separate package. Contributed by Gergely Pollak
This commit is contained in:
parent
2b62b12d0d
commit
5db4c0bf70
|
@ -25,6 +25,7 @@ import org.apache.hadoop.security.Groups;
|
|||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.*;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||
|
@ -74,12 +75,12 @@ public class CSMappingPlacementRule extends PlacementRule {
|
|||
private boolean failOnConfigError = true;
|
||||
|
||||
@VisibleForTesting
|
||||
void setGroups(Groups groups) {
|
||||
public void setGroups(Groups groups) {
|
||||
this.groups = groups;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
void setFailOnConfigError(boolean failOnConfigError) {
|
||||
public void setFailOnConfigError(boolean failOnConfigError) {
|
||||
this.failOnConfigError = failOnConfigError;
|
||||
}
|
||||
|
||||
|
|
|
@ -55,7 +55,7 @@ public class VariableContext {
|
|||
* @param name Name of the variable to check
|
||||
* @return true if the variable is immutable
|
||||
*/
|
||||
boolean isImmutable(String name) {
|
||||
public boolean isImmutable(String name) {
|
||||
return (immutableNames != null && immutableNames.contains(name));
|
||||
}
|
||||
|
||||
|
|
|
@ -16,7 +16,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
|
||||
|
|
@ -16,9 +16,10 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
|
||||
/**
|
||||
* Mapping rule represents a single mapping setting defined by the user. All
|
|
@ -16,9 +16,10 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
|
||||
/**
|
||||
* This interface represents the action part of a MappingRule, action are
|
|
@ -16,7 +16,9 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
|
||||
/**
|
||||
* This class implements the fallback logic for MappingRuleActions, this can
|
|
@ -16,9 +16,10 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
|
||||
/**
|
||||
* This class contains all the actions and some helper methods to generate them.
|
|
@ -16,7 +16,9 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
|
||||
public interface MappingRuleMatcher {
|
||||
/**
|
|
@ -16,7 +16,9 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Set;
|
|
@ -16,7 +16,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
/**
|
||||
* This class represents the outcome of an action.
|
|
@ -16,7 +16,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
public enum MappingRuleResultType {
|
||||
/**
|
|
@ -16,7 +16,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
|
||||
|
@ -30,7 +30,7 @@ import java.util.Set;
|
|||
* interface should contain all validation related data and functions, this way
|
||||
* schedulers or engines can be changed without changing the MappingRules.
|
||||
*/
|
||||
interface MappingRuleValidationContext {
|
||||
public interface MappingRuleValidationContext {
|
||||
/**
|
||||
* This method should determine if the provided queue path can result in
|
||||
* a possible placement. It should fail if the provided path cannot be placed
|
|
@ -15,7 +15,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
|
||||
|
@ -48,7 +48,7 @@ public class MappingRuleValidationContextImpl
|
|||
*/
|
||||
private final CapacitySchedulerQueueManager queueManager;
|
||||
|
||||
MappingRuleValidationContextImpl(CapacitySchedulerQueueManager qm) {
|
||||
public MappingRuleValidationContextImpl(CapacitySchedulerQueueManager qm) {
|
||||
queueManager = qm;
|
||||
}
|
||||
|
|
@ -16,7 +16,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
|
|
@ -18,12 +18,10 @@
|
|||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
|
||||
|
||||
import org.apache.hadoop.ipc.WeightedTimeCostProvider;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueuePlacementRuleUtils;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.MappingRuleCreator;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
|
@ -27,11 +27,11 @@ import java.util.ArrayList;
|
|||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleAction;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleActions;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleMatcher;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleMatchers;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleAction;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleActions;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleMatcher;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleMatchers;
|
||||
|
||||
// These are generated classes - use GeneratePojos class to create them
|
||||
// if they are missing
|
||||
|
|
|
@ -22,11 +22,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
|||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingQueuePath;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingQueuePath;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
|
||||
public class LegacyMappingRuleToJson {
|
||||
//Legacy rule parse helper constants
|
||||
|
|
|
@ -18,6 +18,17 @@
|
|||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
|
@ -29,18 +40,6 @@ import java.util.Optional;
|
|||
import java.util.Set;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
|
||||
|
||||
/**
|
||||
* Maintains a list of queues as well as scheduling parameters for each queue,
|
||||
* such as guaranteed share allocations, from the fair scheduler config file.
|
||||
|
|
|
@ -36,7 +36,7 @@ import java.util.Set;
|
|||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class MockQueueHierarchyBuilder {
|
||||
public class MockQueueHierarchyBuilder {
|
||||
private static final String ROOT = "root";
|
||||
private static final String QUEUE_SEP = ".";
|
||||
private List<String> queuePaths = Lists.newArrayList();
|
||||
|
|
|
@ -24,6 +24,7 @@ import org.apache.hadoop.security.Groups;
|
|||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
|
||||
|
|
|
@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
|
|||
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.TestUserGroupMappingPlacementRule.QueueMappingTestData.QueueMappingTestDataBuilder;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
|
||||
|
|
|
@ -16,7 +16,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
|
||||
|
@ -24,6 +24,9 @@ import org.apache.hadoop.security.Groups;
|
|||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.CSMappingPlacementRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MockQueueHierarchyBuilder;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
|
||||
|
@ -48,12 +51,12 @@ import static junit.framework.TestCase.assertNotNull;
|
|||
import static junit.framework.TestCase.assertNull;
|
||||
import static junit.framework.TestCase.assertTrue;
|
||||
import static junit.framework.TestCase.fail;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.DOT;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class TestCSMappingPlacementRule {
|
||||
public static final String DOT = ".";
|
||||
private static final Logger LOG = LoggerFactory
|
||||
.getLogger(TestCSMappingPlacementRule.class);
|
||||
|
|
@ -16,13 +16,14 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestMappingRule {
|
|
@ -16,7 +16,8 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.junit.Assert.*;
|
|
@ -16,10 +16,11 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
|
||||
import junit.framework.TestCase;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.HashMap;
|
|
@ -16,9 +16,10 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MockQueueHierarchyBuilder;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
|
||||
import org.junit.Test;
|
||||
|
|
@ -16,9 +16,10 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.HashMap;
|
|
@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
|
|||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
|
@ -23,9 +23,9 @@ import static org.junit.Assert.assertEquals;
|
|||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleResult;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleResultType;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleResult;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleResultType;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.MappingRulesDescription;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.Rule;
|
||||
|
|
|
@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placeme
|
|||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||
import org.junit.Test;
|
||||
|
||||
|
|
Loading…
Reference in New Issue