honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Always check that cgroup data is present (#45647) 

`OsProbe` fetches cgroup data from the filesystem, and has asserts that
check for missing values. This PR changes most of these asserts into
runtime checks, since at least one user has reported an NPE where
a piece of cgroup data was missing.

Backport of #45606 to 7.x.
This commit is contained in:
Rory Hunter 2019-08-19 10:29:41 +01:00 committed by GitHub
parent 6f5d944fbd
commit 47b3dccbc4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 125 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
View File

@ -511,17 +511,26 @@ public class OsProbe {
assert !controllerMap.isEmpty();
final String cpuAcctControlGroup = controllerMap.get("cpuacct");
assert cpuAcctControlGroup != null;
if (cpuAcctControlGroup == null) {
logger.debug("no [cpuacct] data found in cgroup stats");
return null;
}
final long cgroupCpuAcctUsageNanos = getCgroupCpuAcctUsageNanos(cpuAcctControlGroup);
final String cpuControlGroup = controllerMap.get("cpu");
assert cpuControlGroup != null;
if (cpuControlGroup == null) {
logger.debug("no [cpu] data found in cgroup stats");
return null;
}
final long cgroupCpuAcctCpuCfsPeriodMicros = getCgroupCpuAcctCpuCfsPeriodMicros(cpuControlGroup);
final long cgroupCpuAcctCpuCfsQuotaMicros = getCgroupCpuAcctCpuCfsQuotaMicros(cpuControlGroup);
final OsStats.Cgroup.CpuStat cpuStat = getCgroupCpuAcctCpuStat(cpuControlGroup);
final String memoryControlGroup = controllerMap.get("memory");
assert memoryControlGroup != null;
if (memoryControlGroup == null) {
logger.debug("no [memory] data found in cgroup stats");
return null;
}
final String cgroupMemoryLimitInBytes = getCgroupMemoryLimitInBytes(memoryControlGroup);
final String cgroupMemoryUsageInBytes = getCgroupMemoryUsageInBytes(memoryControlGroup);

146
server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java
View File

@ -28,6 +28,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.anyOf;
@ -188,22 +189,121 @@ public class OsProbeTests extends ESTestCase {
final boolean areCgroupStatsAvailable = randomBoolean();
final String hierarchy = randomAlphaOfLength(16);
final OsProbe probe = new OsProbe() {
final OsProbe probe = buildStubOsProbe(areCgroupStatsAvailable, hierarchy);
final OsStats.Cgroup cgroup = probe.osStats().getCgroup();
if (areCgroupStatsAvailable) {
assertNotNull(cgroup);
assertThat(cgroup.getCpuAcctControlGroup(), equalTo("/" + hierarchy));
assertThat(cgroup.getCpuAcctUsageNanos(), equalTo(364869866063112L));
assertThat(cgroup.getCpuControlGroup(), equalTo("/" + hierarchy));
assertThat(cgroup.getCpuCfsPeriodMicros(), equalTo(100000L));
assertThat(cgroup.getCpuCfsQuotaMicros(), equalTo(50000L));
assertThat(cgroup.getCpuStat().getNumberOfElapsedPeriods(), equalTo(17992L));
assertThat(cgroup.getCpuStat().getNumberOfTimesThrottled(), equalTo(1311L));
assertThat(cgroup.getCpuStat().getTimeThrottledNanos(), equalTo(139298645489L));
assertThat(cgroup.getMemoryLimitInBytes(), equalTo("18446744073709551615"));
assertThat(cgroup.getMemoryUsageInBytes(), equalTo("4796416"));
} else {
assertNull(cgroup);
}
}
public void testCgroupProbeWithMissingCpuAcct() {
assumeTrue("test runs on Linux only", Constants.LINUX);
final String hierarchy = randomAlphaOfLength(16);
// This cgroup data is missing a line about cpuacct
List<String> procSelfCgroupLines = getProcSelfGroupLines(hierarchy)
.stream()
.map(line -> line.replaceFirst(",cpuacct", ""))
.collect(Collectors.toList());
final OsProbe probe = buildStubOsProbe(true, hierarchy, procSelfCgroupLines);
final OsStats.Cgroup cgroup = probe.osStats().getCgroup();
assertNull(cgroup);
}
public void testCgroupProbeWithMissingCpu() {
assumeTrue("test runs on Linux only", Constants.LINUX);
final String hierarchy = randomAlphaOfLength(16);
// This cgroup data is missing a line about cpu
List<String> procSelfCgroupLines = getProcSelfGroupLines(hierarchy)
.stream()
.map(line -> line.replaceFirst(":cpu,", ":"))
.collect(Collectors.toList());
final OsProbe probe = buildStubOsProbe(true, hierarchy, procSelfCgroupLines);
final OsStats.Cgroup cgroup = probe.osStats().getCgroup();
assertNull(cgroup);
}
public void testCgroupProbeWithMissingMemory() {
assumeTrue("test runs on Linux only", Constants.LINUX);
final String hierarchy = randomAlphaOfLength(16);
// This cgroup data is missing a line about memory
List<String> procSelfCgroupLines = getProcSelfGroupLines(hierarchy)
.stream()
.filter(line -> !line.contains(":memory:"))
.collect(Collectors.toList());
final OsProbe probe = buildStubOsProbe(true, hierarchy, procSelfCgroupLines);
final OsStats.Cgroup cgroup = probe.osStats().getCgroup();
assertNull(cgroup);
}
private static List<String> getProcSelfGroupLines(String hierarchy) {
return Arrays.asList(
"10:freezer:/",
"9:net_cls,net_prio:/",
"8:pids:/",
"7:blkio:/",
"6:memory:/" + hierarchy,
"5:devices:/user.slice",
"4:hugetlb:/",
"3:perf_event:/",
"2:cpu,cpuacct,cpuset:/" + hierarchy,
"1:name=systemd:/user.slice/user-1000.slice/session-2359.scope",
"0::/cgroup2");
}
private static OsProbe buildStubOsProbe(final boolean areCgroupStatsAvailable, final String hierarchy) {
List<String> procSelfCgroupLines = getProcSelfGroupLines(hierarchy);
return buildStubOsProbe(areCgroupStatsAvailable, hierarchy, procSelfCgroupLines);
}
/**
* Builds a test instance of OsProbe. Methods that ordinarily read from the filesystem are overridden to return values based upon
* the arguments to this method.
*
* @param areCgroupStatsAvailable whether or not cgroup data is available. Normally OsProbe establishes this for itself.
* @param hierarchy a mock value used to generate a cgroup hierarchy.
* @param procSelfCgroupLines the lines that will be used as the content of <code>/proc/self/cgroup</code>
* @return a test instance
*/
private static OsProbe buildStubOsProbe(
final boolean areCgroupStatsAvailable,
final String hierarchy,
List<String> procSelfCgroupLines
) {
return new OsProbe() {
@Override
List<String> readProcSelfCgroup() {
return Arrays.asList(
"10:freezer:/",
"9:net_cls,net_prio:/",
"8:pids:/",
"7:blkio:/",
"6:memory:/" + hierarchy,
"5:devices:/user.slice",
"4:hugetlb:/",
"3:perf_event:/",
"2:cpu,cpuacct,cpuset:/" + hierarchy,
"1:name=systemd:/user.slice/user-1000.slice/session-2359.scope",
"0::/cgroup2");
return procSelfCgroupLines;
}
@Override
@ -249,26 +349,6 @@ public class OsProbeTests extends ESTestCase {
boolean areCgroupStatsAvailable() {
return areCgroupStatsAvailable;
}
};
final OsStats.Cgroup cgroup = probe.osStats().getCgroup();
if (areCgroupStatsAvailable) {
assertNotNull(cgroup);
assertThat(cgroup.getCpuAcctControlGroup(), equalTo("/" + hierarchy));
assertThat(cgroup.getCpuAcctUsageNanos(), equalTo(364869866063112L));
assertThat(cgroup.getCpuControlGroup(), equalTo("/" + hierarchy));
assertThat(cgroup.getCpuCfsPeriodMicros(), equalTo(100000L));
assertThat(cgroup.getCpuCfsQuotaMicros(), equalTo(50000L));
assertThat(cgroup.getCpuStat().getNumberOfElapsedPeriods(), equalTo(17992L));
assertThat(cgroup.getCpuStat().getNumberOfTimesThrottled(), equalTo(1311L));
assertThat(cgroup.getCpuStat().getTimeThrottledNanos(), equalTo(139298645489L));
assertThat(cgroup.getMemoryLimitInBytes(), equalTo("18446744073709551615"));
assertThat(cgroup.getMemoryUsageInBytes(), equalTo("4796416"));
} else {
assertNull(cgroup);
}
}
}