YARN-10412. Move CS placement rule related changes to a separate package. Contributed by Gergely Pollak

This commit is contained in:
Szilard Nemeth 2021-03-12 14:10:16 +01:00
parent 2b62b12d0d
commit 5db4c0bf70
30 changed files with 144 additions and 128 deletions

View File

@ -25,6 +25,7 @@ import org.apache.hadoop.security.Groups;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.*;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@ -74,12 +75,12 @@ public class CSMappingPlacementRule extends PlacementRule {
private boolean failOnConfigError = true;
@VisibleForTesting
void setGroups(Groups groups) {
public void setGroups(Groups groups) {
this.groups = groups;
}
@VisibleForTesting
void setFailOnConfigError(boolean failOnConfigError) {
public void setFailOnConfigError(boolean failOnConfigError) {
this.failOnConfigError = failOnConfigError;
}

View File

@ -55,7 +55,7 @@ public class VariableContext {
* @param name Name of the variable to check
* @return true if the variable is immutable
*/
boolean isImmutable(String name) {
public boolean isImmutable(String name) {
return (immutableNames != null && immutableNames.contains(name));
}

View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;

View File

@ -16,9 +16,10 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
/**
* Mapping rule represents a single mapping setting defined by the user. All

View File

@ -16,9 +16,10 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
/**
* This interface represents the action part of a MappingRule, action are

View File

@ -16,7 +16,9 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
/**
* This class implements the fallback logic for MappingRuleActions, this can

View File

@ -16,9 +16,10 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
/**
* This class contains all the actions and some helper methods to generate them.

View File

@ -16,7 +16,9 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
public interface MappingRuleMatcher {
/**

View File

@ -16,7 +16,9 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
import java.util.Arrays;
import java.util.Set;

View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
/**
* This class represents the outcome of an action.

View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import org.apache.hadoop.yarn.exceptions.YarnException;
@ -30,7 +30,7 @@ import java.util.Set;
* interface should contain all validation related data and functions, this way
* schedulers or engines can be changed without changing the MappingRules.
*/
interface MappingRuleValidationContext {
public interface MappingRuleValidationContext {
/**
* This method should determine if the provided queue path can result in
* a possible placement. It should fail if the provided path cannot be placed

View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
@ -48,7 +48,7 @@ public class MappingRuleValidationContextImpl
*/
private final CapacitySchedulerQueueManager queueManager;
MappingRuleValidationContextImpl(CapacitySchedulerQueueManager qm) {
public MappingRuleValidationContextImpl(CapacitySchedulerQueueManager qm) {
queueManager = qm;
}

View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;

View File

@ -18,12 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import org.apache.hadoop.ipc.WeightedTimeCostProvider;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueuePlacementRuleUtils;
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.MappingRuleCreator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

View File

@ -27,11 +27,11 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleAction;
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleActions;
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleMatcher;
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleMatchers;
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleAction;
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleActions;
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleMatcher;
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleMatchers;
// These are generated classes - use GeneratePojos class to create them
// if they are missing

View File

@ -22,11 +22,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingQueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingQueuePath;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
public class LegacyMappingRuleToJson {
//Legacy rule parse helper constants

View File

@ -18,6 +18,17 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@ -29,18 +40,6 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
/**
* Maintains a list of queues as well as scheduling parameters for each queue,
* such as guaranteed share allocations, from the fair scheduler config file.

View File

@ -36,7 +36,7 @@ import java.util.Set;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
class MockQueueHierarchyBuilder {
public class MockQueueHierarchyBuilder {
private static final String ROOT = "root";
private static final String QUEUE_SEP = ".";
private List<String> queuePaths = Lists.newArrayList();

View File

@ -24,6 +24,7 @@ import org.apache.hadoop.security.Groups;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;

View File

@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
import org.apache.hadoop.yarn.server.resourcemanager.placement.TestUserGroupMappingPlacementRule.QueueMappingTestData.QueueMappingTestDataBuilder;
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;

View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
@ -24,6 +24,9 @@ import org.apache.hadoop.security.Groups;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
import org.apache.hadoop.yarn.server.resourcemanager.placement.CSMappingPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.MockQueueHierarchyBuilder;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
@ -48,12 +51,12 @@ import static junit.framework.TestCase.assertNotNull;
import static junit.framework.TestCase.assertNull;
import static junit.framework.TestCase.assertTrue;
import static junit.framework.TestCase.fail;
import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.DOT;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class TestCSMappingPlacementRule {
public static final String DOT = ".";
private static final Logger LOG = LoggerFactory
.getLogger(TestCSMappingPlacementRule.class);

View File

@ -16,13 +16,14 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
import org.junit.Test;
public class TestMappingRule {

View File

@ -16,7 +16,8 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
import org.junit.Test;
import static org.junit.Assert.*;

View File

@ -16,10 +16,11 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
import junit.framework.TestCase;
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
import org.junit.Test;
import java.util.HashMap;

View File

@ -16,9 +16,10 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.MockQueueHierarchyBuilder;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
import org.junit.Test;

View File

@ -16,9 +16,10 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
import org.junit.Test;
import java.util.HashMap;

View File

@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
import java.util.List;
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

View File

@ -23,9 +23,9 @@ import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleResult;
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleResultType;
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleResult;
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleResultType;
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.MappingRulesDescription;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.Rule;

View File

@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placeme
import static org.junit.Assert.*;
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.junit.Test;