YARN-10386. Create new JSON schema for Placement Rules. Contributed by Peter Bacsko and Benjamin Teke

This commit is contained in:
Szilard Nemeth 2020-08-28 17:09:31 +02:00
parent c392d9022a
commit 6e3d7050a9
7 changed files with 221 additions and 0 deletions

1
.gitignore vendored
View File

@ -51,6 +51,7 @@ patchprocess/
.history/
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package-lock.json
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn-error.log
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/schema
#robotframework outputs
log.html

View File

@ -207,6 +207,7 @@
<powermock.version>1.5.6</powermock.version>
<solr.version>7.7.0</solr.version>
<openssl-wildfly.version>1.0.7.Final</openssl-wildfly.version>
<jsonschema2pojo.version>1.0.2</jsonschema2pojo.version>
</properties>
<dependencyManagement>
@ -1165,6 +1166,11 @@
<artifactId>jackson-dataformat-cbor</artifactId>
<version>${jackson2.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-yaml</artifactId>
<version>${jackson2.version}</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
@ -1728,6 +1734,49 @@
<artifactId>jna</artifactId>
<version>${jna.version}</version>
</dependency>
<dependency>
<groupId>org.jsonschema2pojo</groupId>
<artifactId>jsonschema2pojo-core</artifactId>
<version>${jsonschema2pojo.version}</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>annotations</artifactId>
</exclusion>
<exclusion>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</exclusion>
<exclusion>
<groupId>org.jsonschema2pojo</groupId>
<artifactId>jsonschema2pojo-scalagen</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.javaparser</groupId>
<artifactId>javaparser</artifactId>
</exclusion>
<exclusion>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</dependencyManagement>

View File

@ -711,4 +711,9 @@
<Class name="~org\.apache\.hadoop\.yarn\.server\.timelineservice\.reader\.TestTimelineReaderWebServicesHBaseStorage.*" />
<Bug pattern="UMAC_UNCALLABLE_METHOD_OF_ANONYMOUS_CLASS" />
</Match>
<!-- Generated files from JSON schema -->
<Match>
<Package name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema" />
</Match>
</FindBugsFilter>

View File

@ -250,6 +250,21 @@
<scope>test</scope>
</dependency>
<!-- Necessary to include jackson separately to avoid issues with
maven enforcer plugin in "hadoop-client-check-test-invariants"
-->
<dependency>
<groupId>org.jsonschema2pojo</groupId>
<artifactId>jsonschema2pojo-core</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-yaml</artifactId>
</dependency>
</dependencies>
<build>
@ -380,6 +395,22 @@
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.jsonschema2pojo</groupId>
<artifactId>jsonschema2pojo-maven-plugin</artifactId>
<configuration>
<sourceDirectory>${basedir}/src/main/json_schema</sourceDirectory>
<targetPackage>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema</targetPackage>
<outputDirectory>${basedir}/src/main/java</outputDirectory>
</configuration>
<executions>
<execution>
<goals>
<goal>generate</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

View File

@ -0,0 +1,76 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Paths;
import org.jsonschema2pojo.DefaultGenerationConfig;
import org.jsonschema2pojo.GenerationConfig;
import org.jsonschema2pojo.Jackson2Annotator;
import org.jsonschema2pojo.SchemaGenerator;
import org.jsonschema2pojo.SchemaMapper;
import org.jsonschema2pojo.SchemaStore;
import org.jsonschema2pojo.rules.RuleFactory;
import com.sun.codemodel.JCodeModel;
/**
* Helper class to re-generate java POJOs based on the JSON schema.
*/
public final class GeneratePojos {
@SuppressWarnings("checkstyle:linelength")
private static final String TARGET_PACKAGE =
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema";
private GeneratePojos() {
// no instances
}
public static void main(String[] args) throws IOException {
JCodeModel codeModel = new JCodeModel();
URL schemaURL = Paths.get(
"src/main/json_schema/MappingRulesDescription.json").toUri().toURL();
GenerationConfig config = new DefaultGenerationConfig() {
@Override
public boolean isGenerateBuilders() {
return false;
}
@Override
public boolean isUsePrimitives() {
return true;
}
};
SchemaMapper mapper =
new SchemaMapper(
new RuleFactory(config,
new Jackson2Annotator(config),
new SchemaStore()),
new SchemaGenerator());
mapper.generate(codeModel, "ignore", TARGET_PACKAGE, schemaURL);
codeModel.build(new File("src/main/java"));
}
}

View File

@ -0,0 +1,23 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Contains classes that are related to the newer
* version of CS placement engine.
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement;

View File

@ -0,0 +1,36 @@
{
"$schema": "http://json-schema.org/draft/2019-09/schema#",
"type" : "object",
"properties" : {
"rules" : {
"type" : "array",
"items": {
"type": "object",
"properties": {
"type": { "type": "string",
"enum": ["user", "group", "application"] },
"matches": { "type": "string" },
"policy": { "type": "string",
"enum": [
"specified",
"reject",
"defaultQueue",
"user",
"primaryGroup",
"secondaryGroup",
"primaryGroupUser",
"secondaryGroupUser",
"setDefaultQueue",
"custom" ] },
"parentQueue": { "type": "string" },
"fallbackResult": { "type": "string",
"enum": ["skip", "reject", "placeDefault"] },
"create": { "type": "boolean" },
"value": { "type": "string" },
"customPlacement": { "type" : "string" }
}
}
}
}
}