YARN-9135. NM State store ResourceMappings serialization are tested with Strings instead of real Device objects. Contributed by Peter Bacsko

This commit is contained in:
Szilard Nemeth 2019-08-12 14:02:17 +02:00
parent 330e450397
commit b20fd9e212
11 changed files with 517 additions and 57 deletions

View File

@ -0,0 +1 @@
../../target/generated-sources/vendor/apidoc/bin/apidoc

View File

@ -0,0 +1 @@
../../target/generated-sources/vendor/markdown-it/bin/markdown-it.js

View File

@ -0,0 +1 @@
../../target/generated-sources/vendor/requirejs/bin/r.js

View File

@ -0,0 +1 @@
../../target/generated-sources/vendor/requirejs/bin/r.js

View File

@ -0,0 +1 @@
../../target/generated-sources/vendor/semver/bin/semver

View File

@ -0,0 +1 @@
../../target/generated-sources/vendor/shelljs/bin/shjs

View File

@ -0,0 +1,422 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1
angular-loader@~1.6.4:
version "1.6.10"
resolved "https://registry.yarnpkg.com/angular-loader/-/angular-loader-1.6.10.tgz#5ab1995c65dd38640e6d9d738dbd85eddd5ec310"
angular-mocks@~1.6.4:
version "1.6.10"
resolved "https://registry.yarnpkg.com/angular-mocks/-/angular-mocks-1.6.10.tgz#6a139e43c461d0c9a5a1acebc91e63db16031176"
angular-route@~1.6.4:
version "1.6.10"
resolved "https://registry.yarnpkg.com/angular-route/-/angular-route-1.6.10.tgz#4247a32eab19495624623e96c1626dfba17ebf21"
angular@~1.6.4:
version "1.6.10"
resolved "https://registry.yarnpkg.com/angular/-/angular-1.6.10.tgz#eed3080a34d29d0f681ff119b18ce294e3f74826"
apidoc-core@~0.8.2:
version "0.8.3"
resolved "https://registry.yarnpkg.com/apidoc-core/-/apidoc-core-0.8.3.tgz#d9d63545829df250d2cca049683a87e775364b96"
dependencies:
fs-extra "^3.0.1"
glob "^7.1.1"
iconv-lite "^0.4.17"
klaw-sync "^2.1.0"
lodash "~4.17.4"
semver "~5.3.0"
apidoc@0.17.7:
version "0.17.7"
resolved "https://registry.yarnpkg.com/apidoc/-/apidoc-0.17.7.tgz#a49090cbd8b3aa457bd054f00e0037e975fd3ee7"
dependencies:
apidoc-core "~0.8.2"
commander "^2.19.0"
fs-extra "^7.0.0"
lodash "^4.17.10"
markdown-it "^8.3.1"
winston "^3.0.0"
argparse@^1.0.7:
version "1.0.10"
resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
dependencies:
sprintf-js "~1.0.2"
async@^2.6.1:
version "2.6.3"
resolved "https://registry.yarnpkg.com/async/-/async-2.6.3.tgz#d72625e2344a3656e3a3ad4fa749fa83299d82ff"
dependencies:
lodash "^4.17.14"
balanced-match@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
bootstrap@~3.3.7:
version "3.3.7"
resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-3.3.7.tgz#5a389394549f23330875a3b150656574f8a9eb71"
brace-expansion@^1.1.7:
version "1.1.11"
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
dependencies:
balanced-match "^1.0.0"
concat-map "0.0.1"
color-convert@^1.9.1:
version "1.9.3"
resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"
dependencies:
color-name "1.1.3"
color-name@1.1.3:
version "1.1.3"
resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25"
color-name@^1.0.0:
version "1.1.4"
resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
color-string@^1.5.2:
version "1.5.3"
resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.5.3.tgz#c9bbc5f01b58b5492f3d6857459cb6590ce204cc"
dependencies:
color-name "^1.0.0"
simple-swizzle "^0.2.2"
color@3.0.x:
version "3.0.0"
resolved "https://registry.yarnpkg.com/color/-/color-3.0.0.tgz#d920b4328d534a3ac8295d68f7bd4ba6c427be9a"
dependencies:
color-convert "^1.9.1"
color-string "^1.5.2"
colornames@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/colornames/-/colornames-1.1.1.tgz#f8889030685c7c4ff9e2a559f5077eb76a816f96"
colors@^1.2.1:
version "1.3.3"
resolved "https://registry.yarnpkg.com/colors/-/colors-1.3.3.tgz#39e005d546afe01e01f9c4ca8fa50f686a01205d"
colorspace@1.1.x:
version "1.1.2"
resolved "https://registry.yarnpkg.com/colorspace/-/colorspace-1.1.2.tgz#e0128950d082b86a2168580796a0aa5d6c68d8c5"
dependencies:
color "3.0.x"
text-hex "1.0.x"
commander@^2.19.0:
version "2.20.0"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.0.tgz#d58bb2b5c1ee8f87b0d340027e9e94e222c5a422"
concat-map@0.0.1:
version "0.0.1"
resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
core-util-is@~1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
diagnostics@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/diagnostics/-/diagnostics-1.1.1.tgz#cab6ac33df70c9d9a727490ae43ac995a769b22a"
dependencies:
colorspace "1.1.x"
enabled "1.0.x"
kuler "1.0.x"
enabled@1.0.x:
version "1.0.2"
resolved "https://registry.yarnpkg.com/enabled/-/enabled-1.0.2.tgz#965f6513d2c2d1c5f4652b64a2e3396467fc2f93"
dependencies:
env-variable "0.0.x"
entities@~1.1.1:
version "1.1.2"
resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56"
env-variable@0.0.x:
version "0.0.5"
resolved "https://registry.yarnpkg.com/env-variable/-/env-variable-0.0.5.tgz#913dd830bef11e96a039c038d4130604eba37f88"
fast-safe-stringify@^2.0.4:
version "2.0.6"
resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.6.tgz#04b26106cc56681f51a044cfc0d76cf0008ac2c2"
fecha@^2.3.3:
version "2.3.3"
resolved "https://registry.yarnpkg.com/fecha/-/fecha-2.3.3.tgz#948e74157df1a32fd1b12c3a3c3cdcb6ec9d96cd"
fs-extra@^3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-3.0.1.tgz#3794f378c58b342ea7dbbb23095109c4b3b62291"
dependencies:
graceful-fs "^4.1.2"
jsonfile "^3.0.0"
universalify "^0.1.0"
fs-extra@^7.0.0:
version "7.0.1"
resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-7.0.1.tgz#4f189c44aa123b895f722804f55ea23eadc348e9"
dependencies:
graceful-fs "^4.1.2"
jsonfile "^4.0.0"
universalify "^0.1.0"
fs.realpath@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
glob@^7.1.1:
version "7.1.4"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.4.tgz#aa608a2f6c577ad357e1ae5a5c26d9a8d1969255"
dependencies:
fs.realpath "^1.0.0"
inflight "^1.0.4"
inherits "2"
minimatch "^3.0.4"
once "^1.3.0"
path-is-absolute "^1.0.0"
graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6:
version "4.2.0"
resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.0.tgz#8d8fdc73977cb04104721cb53666c1ca64cd328b"
iconv-lite@^0.4.17:
version "0.4.24"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
dependencies:
safer-buffer ">= 2.1.2 < 3"
inflight@^1.0.4:
version "1.0.6"
resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9"
dependencies:
once "^1.3.0"
wrappy "1"
inherits@2, inherits@^2.0.3, inherits@~2.0.3:
version "2.0.4"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
is-arrayish@^0.3.1:
version "0.3.2"
resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03"
is-stream@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
isarray@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
jquery@3.3.1:
version "3.3.1"
resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.3.1.tgz#958ce29e81c9790f31be7792df5d4d95fc57fbca"
jsonfile@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-3.0.1.tgz#a5ecc6f65f53f662c4415c7675a0331d0992ec66"
optionalDependencies:
graceful-fs "^4.1.6"
jsonfile@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb"
optionalDependencies:
graceful-fs "^4.1.6"
klaw-sync@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/klaw-sync/-/klaw-sync-2.1.0.tgz#3d3bcd8600e7bfdef53231c739ff053aed560e44"
optionalDependencies:
graceful-fs "^4.1.11"
kuler@1.0.x:
version "1.0.1"
resolved "https://registry.yarnpkg.com/kuler/-/kuler-1.0.1.tgz#ef7c784f36c9fb6e16dd3150d152677b2b0228a6"
dependencies:
colornames "^1.1.1"
linkify-it@^2.0.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-2.2.0.tgz#e3b54697e78bf915c70a38acd78fd09e0058b1cf"
dependencies:
uc.micro "^1.0.1"
lodash@^4.17.10, lodash@^4.17.14, lodash@~4.17.4:
version "4.17.14"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.14.tgz#9ce487ae66c96254fe20b599f21b6816028078ba"
logform@^2.1.1:
version "2.1.2"
resolved "https://registry.yarnpkg.com/logform/-/logform-2.1.2.tgz#957155ebeb67a13164069825ce67ddb5bb2dd360"
dependencies:
colors "^1.2.1"
fast-safe-stringify "^2.0.4"
fecha "^2.3.3"
ms "^2.1.1"
triple-beam "^1.3.0"
markdown-it@^8.3.1:
version "8.4.2"
resolved "https://registry.yarnpkg.com/markdown-it/-/markdown-it-8.4.2.tgz#386f98998dc15a37722aa7722084f4020bdd9b54"
dependencies:
argparse "^1.0.7"
entities "~1.1.1"
linkify-it "^2.0.0"
mdurl "^1.0.1"
uc.micro "^1.0.5"
mdurl@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-1.0.1.tgz#fe85b2ec75a59037f2adfec100fd6c601761152e"
minimatch@^3.0.4:
version "3.0.4"
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
dependencies:
brace-expansion "^1.1.7"
ms@^2.1.1:
version "2.1.2"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
once@^1.3.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
dependencies:
wrappy "1"
one-time@0.0.4:
version "0.0.4"
resolved "https://registry.yarnpkg.com/one-time/-/one-time-0.0.4.tgz#f8cdf77884826fe4dff93e3a9cc37b1e4480742e"
path-is-absolute@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f"
process-nextick-args@~2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
readable-stream@^2.3.6:
version "2.3.6"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf"
dependencies:
core-util-is "~1.0.0"
inherits "~2.0.3"
isarray "~1.0.0"
process-nextick-args "~2.0.0"
safe-buffer "~5.1.1"
string_decoder "~1.1.1"
util-deprecate "~1.0.1"
readable-stream@^3.1.1:
version "3.4.0"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.4.0.tgz#a51c26754658e0a3c21dbf59163bd45ba6f447fc"
dependencies:
inherits "^2.0.3"
string_decoder "^1.1.1"
util-deprecate "^1.0.1"
requirejs@^2.1.0:
version "2.3.6"
resolved "https://registry.yarnpkg.com/requirejs/-/requirejs-2.3.6.tgz#e5093d9601c2829251258c0b9445d4d19fa9e7c9"
roboto-fontface@0.10.0:
version "0.10.0"
resolved "https://registry.yarnpkg.com/roboto-fontface/-/roboto-fontface-0.10.0.tgz#7eee40cfa18b1f7e4e605eaf1a2740afb6fd71b0"
safe-buffer@~5.1.0, safe-buffer@~5.1.1:
version "5.1.2"
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
"safer-buffer@>= 2.1.2 < 3":
version "2.1.2"
resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
semver@~5.3.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f"
shelljs@^0.2.6:
version "0.2.6"
resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.2.6.tgz#90492d72ffcc8159976baba62fb0f6884f0c3378"
simple-swizzle@^0.2.2:
version "0.2.2"
resolved "https://registry.yarnpkg.com/simple-swizzle/-/simple-swizzle-0.2.2.tgz#a4da6b635ffcccca33f70d17cb92592de95e557a"
dependencies:
is-arrayish "^0.3.1"
sprintf-js@~1.0.2:
version "1.0.3"
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
stack-trace@0.0.x:
version "0.0.10"
resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0"
string_decoder@^1.1.1:
version "1.2.0"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.2.0.tgz#fe86e738b19544afe70469243b2a1ee9240eae8d"
dependencies:
safe-buffer "~5.1.0"
string_decoder@~1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
dependencies:
safe-buffer "~5.1.0"
text-hex@1.0.x:
version "1.0.0"
resolved "https://registry.yarnpkg.com/text-hex/-/text-hex-1.0.0.tgz#69dc9c1b17446ee79a92bf5b884bb4b9127506f5"
triple-beam@^1.2.0, triple-beam@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9"
uc.micro@^1.0.1, uc.micro@^1.0.5:
version "1.0.6"
resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac"
universalify@^0.1.0:
version "0.1.2"
resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"
util-deprecate@^1.0.1, util-deprecate@~1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
winston-transport@^4.3.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.3.0.tgz#df68c0c202482c448d9b47313c07304c2d7c2c66"
dependencies:
readable-stream "^2.3.6"
triple-beam "^1.2.0"
winston@^3.0.0:
version "3.2.1"
resolved "https://registry.yarnpkg.com/winston/-/winston-3.2.1.tgz#63061377976c73584028be2490a1846055f77f07"
dependencies:
async "^2.6.1"
diagnostics "^1.1.1"
is-stream "^1.1.0"
logform "^2.1.1"
one-time "0.0.4"
readable-stream "^3.1.1"
stack-trace "0.0.x"
triple-beam "^1.3.0"
winston-transport "^4.3.0"
wrappy@1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"

View File

@ -17,9 +17,11 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.numa;
import com.google.common.collect.ImmutableMap;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
/**
@ -28,27 +30,18 @@ import java.util.Set;
*/
public class NumaResourceAllocation implements Serializable {
private static final long serialVersionUID = 6339719798446595123L;
private Map<String, Long> nodeVsMemory;
private Map<String, Integer> nodeVsCpus;
private final ImmutableMap<String, Long> nodeVsMemory;
private final ImmutableMap<String, Integer> nodeVsCpus;
public NumaResourceAllocation() {
nodeVsMemory = new HashMap<>();
nodeVsCpus = new HashMap<>();
public NumaResourceAllocation(Map<String, Long> memoryAllocations,
Map<String, Integer> cpuAllocations) {
nodeVsMemory = ImmutableMap.copyOf(memoryAllocations);
nodeVsCpus = ImmutableMap.copyOf(cpuAllocations);
}
public NumaResourceAllocation(String memNodeId, long memory, String cpuNodeId,
int cpus) {
this();
nodeVsMemory.put(memNodeId, memory);
nodeVsCpus.put(cpuNodeId, cpus);
}
public void addMemoryNode(String memNodeId, long memory) {
nodeVsMemory.put(memNodeId, memory);
}
public void addCpuNode(String cpuNodeId, int cpus) {
nodeVsCpus.put(cpuNodeId, cpus);
this(ImmutableMap.of(memNodeId, memory), ImmutableMap.of(cpuNodeId, cpus));
}
public Set<String> getMemNodes() {
@ -59,11 +52,37 @@ public class NumaResourceAllocation implements Serializable {
return nodeVsCpus.keySet();
}
public Map<String, Long> getNodeVsMemory() {
public ImmutableMap<String, Long> getNodeVsMemory() {
return nodeVsMemory;
}
public Map<String, Integer> getNodeVsCpus() {
public ImmutableMap<String, Integer> getNodeVsCpus() {
return nodeVsCpus;
}
@Override
public String toString() {
return "NumaResourceAllocation{" +
"nodeVsMemory=" + nodeVsMemory +
", nodeVsCpus=" + nodeVsCpus +
'}';
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
NumaResourceAllocation that = (NumaResourceAllocation) o;
return Objects.equals(nodeVsMemory, that.nodeVsMemory) &&
Objects.equals(nodeVsCpus, that.nodeVsCpus);
}
@Override
public int hashCode() {
return Objects.hash(nodeVsMemory, nodeVsCpus);
}
}

View File

@ -31,6 +31,7 @@ import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.util.StringUtils;
@ -247,17 +248,19 @@ public class NumaResourceAllocator {
// If there is no single node matched for the container resource
// Check the NUMA nodes for Memory resources
NumaResourceAllocation assignedNumaNodeInfo = new NumaResourceAllocation();
long memreq = resource.getMemorySize();
long memoryRequirement = resource.getMemorySize();
Map<String, Long> memoryAllocations = Maps.newHashMap();
for (NumaNodeResource numaNode : numaNodesList) {
long memrem = numaNode.assignAvailableMemory(memreq, containerId);
assignedNumaNodeInfo.addMemoryNode(numaNode.getNodeId(), memreq - memrem);
memreq = memrem;
if (memreq == 0) {
long memoryRemaining = numaNode.
assignAvailableMemory(memoryRequirement, containerId);
memoryAllocations.put(numaNode.getNodeId(),
memoryRequirement - memoryRemaining);
memoryRequirement = memoryRemaining;
if (memoryRequirement == 0) {
break;
}
}
if (memreq != 0) {
if (memoryRequirement != 0) {
LOG.info("There is no available memory:" + resource.getMemorySize()
+ " in numa nodes for " + containerId);
releaseNumaResource(containerId);
@ -265,26 +268,31 @@ public class NumaResourceAllocator {
}
// Check the NUMA nodes for CPU resources
int cpusreq = resource.getVirtualCores();
int cpusRequirement = resource.getVirtualCores();
Map<String, Integer> cpuAllocations = Maps.newHashMap();
for (int index = 0; index < numaNodesList.size(); index++) {
NumaNodeResource numaNode = numaNodesList
.get((currentAssignNode + index) % numaNodesList.size());
int cpusrem = numaNode.assignAvailableCpus(cpusreq, containerId);
assignedNumaNodeInfo.addCpuNode(numaNode.getNodeId(), cpusreq - cpusrem);
cpusreq = cpusrem;
if (cpusreq == 0) {
int cpusRemaining = numaNode.
assignAvailableCpus(cpusRequirement, containerId);
cpuAllocations.put(numaNode.getNodeId(), cpusRequirement - cpusRemaining);
cpusRequirement = cpusRemaining;
if (cpusRequirement == 0) {
currentAssignNode = (currentAssignNode + index + 1)
% numaNodesList.size();
break;
}
}
if (cpusreq != 0) {
if (cpusRequirement != 0) {
LOG.info("There are no available cpus:" + resource.getVirtualCores()
+ " in numa nodes for " + containerId);
releaseNumaResource(containerId);
return null;
}
NumaResourceAllocation assignedNumaNodeInfo =
new NumaResourceAllocation(memoryAllocations, cpuAllocations);
LOG.info("Assigning multiple NUMA nodes ("
+ StringUtils.join(",", assignedNumaNodeInfo.getMemNodes())
+ ") for memory, ("

View File

@ -1459,8 +1459,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
String keyResChng = CONTAINERS_KEY_PREFIX + container.getContainerId().toString()
+ CONTAINER_ASSIGNED_RESOURCES_KEY_SUFFIX + resourceType;
try {
WriteBatch batch = db.createWriteBatch();
try {
try (WriteBatch batch = db.createWriteBatch()) {
ResourceMappings.AssignedResources res =
new ResourceMappings.AssignedResources();
res.updateAssignedResources(assignedResources);
@ -1468,8 +1467,6 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
// New value will overwrite old values for the same key
batch.put(bytes(keyResChng), res.toBytes());
db.write(batch);
} finally {
batch.close();
}
} catch (DBException e) {
markStoreUnHealthy(e);

View File

@ -75,6 +75,9 @@ import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.nodemanager.amrmproxy.AMRMProxyTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.numa.NumaResourceAllocation;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.LocalResourceTrackerState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredAMRMProxyState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredApplicationsState;
@ -1448,7 +1451,7 @@ public class TestNMLeveldbStateStoreService {
@Test
public void testStateStoreForResourceMapping() throws IOException {
// test empty when no state
// test that stateStore is initially empty
List<RecoveredContainerState> recoveredContainers =
loadContainersState(stateStore.getContainerStateIterator());
assertTrue(recoveredContainers.isEmpty());
@ -1464,38 +1467,43 @@ public class TestNMLeveldbStateStoreService {
ResourceMappings resourceMappings = new ResourceMappings();
when(container.getResourceMappings()).thenReturn(resourceMappings);
// Store ResourceMapping
stateStore.storeAssignedResources(container, "gpu",
Arrays.asList("1", "2", "3"));
// This will overwrite above
List<Serializable> gpuRes1 = Arrays.asList("1", "2", "4");
Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 2),
new GpuDevice(3, 3)));
// This will overwrite the above
List<Serializable> gpuRes1 = Arrays.asList(
new GpuDevice(1, 1), new GpuDevice(2, 2), new GpuDevice(4, 4));
stateStore.storeAssignedResources(container, "gpu", gpuRes1);
List<Serializable> fpgaRes = Arrays.asList("3", "4", "5", "6");
List<Serializable> fpgaRes = Arrays.asList(
new FpgaDevice("testType", 3, 3, "testIPID"),
new FpgaDevice("testType", 4, 4, "testIPID"),
new FpgaDevice("testType", 5, 5, "testIPID"),
new FpgaDevice("testType", 6, 6, "testIPID"));
stateStore.storeAssignedResources(container, "fpga", fpgaRes);
List<Serializable> numaRes = Arrays.asList("numa1");
List<Serializable> numaRes = Arrays.asList(
new NumaResourceAllocation("testmemNodeId", 2048, "testCpuNodeId", 10));
stateStore.storeAssignedResources(container, "numa", numaRes);
// add a invalid key
restartStateStore();
recoveredContainers =
loadContainersState(stateStore.getContainerStateIterator());
assertEquals(1, recoveredContainers.size());
RecoveredContainerState rcs = recoveredContainers.get(0);
List<Serializable> res = rcs.getResourceMappings()
List<Serializable> resources = rcs.getResourceMappings()
.getAssignedResources("gpu");
Assert.assertTrue(res.equals(gpuRes1));
Assert.assertTrue(
resourceMappings.getAssignedResources("gpu").equals(gpuRes1));
Assert.assertEquals(gpuRes1, resources);
Assert.assertEquals(gpuRes1, resourceMappings.getAssignedResources("gpu"));
res = rcs.getResourceMappings().getAssignedResources("fpga");
Assert.assertTrue(res.equals(fpgaRes));
Assert.assertTrue(
resourceMappings.getAssignedResources("fpga").equals(fpgaRes));
resources = rcs.getResourceMappings().getAssignedResources("fpga");
Assert.assertEquals(fpgaRes, resources);
Assert.assertEquals(fpgaRes, resourceMappings.getAssignedResources("fpga"));
res = rcs.getResourceMappings().getAssignedResources("numa");
Assert.assertTrue(res.equals(numaRes));
Assert.assertTrue(
resourceMappings.getAssignedResources("numa").equals(numaRes));
resources = rcs.getResourceMappings().getAssignedResources("numa");
Assert.assertEquals(numaRes, resources);
Assert.assertEquals(numaRes, resourceMappings.getAssignedResources("numa"));
}
@Test