honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add more detailed OS name on Linux (#35352) 

Today our OS information returned in node stats only returns a
high-level name of the OS (e.g., "Linux"). Yet, for some uses this is
too high-level and knowing at a finer level of granularity the
underlying OS can be useful. This commit extracts the pretty name on
Linux from /etc/os-release. This pretty name usually includes the Linux
vendor and the Linux vendor version number (e.g., Fedora 28).
This commit is contained in:
Jason Tedor 2018-11-08 12:16:58 -05:00 committed by GitHub
parent 85a8b517bd
commit 730ec1ddfb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 240 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/reference/cluster/stats.asciidoc
View File

@ -123,6 +123,12 @@ Will return, for example:
"count": 1
}
],
"pretty_names": [
{
"pretty_name": "Mac OS X",
"count": 1
}
],
"mem" : {
"total" : "16gb",
"total_in_bytes" : 17179869184,

55
qa/evil-tests/src/test/java/org/elasticsearch/monitor/os/EvilOsProbeTests.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.monitor.os;
import org.apache.lucene.util.Constants;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.nio.file.Files;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.hamcrest.Matchers.equalTo;
public class EvilOsProbeTests extends ESTestCase {
public void testOsPrettyName() throws IOException {
final OsInfo osInfo = OsProbe.getInstance().osInfo(randomLongBetween(1, 100), randomIntBetween(1, 8));
if (Constants.LINUX) {
final List<String> lines = Files.readAllLines(PathUtils.get("/etc/os-release"));
for (final String line : lines) {
if (line != null && line.startsWith("PRETTY_NAME=")) {
final Matcher matcher = Pattern.compile("PRETTY_NAME=(\"?|'?)?([^\"']+)\\1").matcher(line);
assert matcher.matches() : line;
final String prettyName = matcher.group(2);
assertThat(osInfo.getPrettyName(), equalTo(prettyName));
return;
}
}
assertThat(osInfo.getPrettyName(), equalTo("Linux"));
} else {
assertThat(osInfo.getPrettyName(), equalTo(Constants.OS_NAME));
}
}
}

23
server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java
View File

@ -226,6 +226,7 @@ public class ClusterStatsNodes implements ToXContentFragment {
final int availableProcessors;
final int allocatedProcessors;
final ObjectIntHashMap<String> names;
final ObjectIntHashMap<String> prettyNames;
final org.elasticsearch.monitor.os.OsStats.Mem mem;
/**
@ -233,6 +234,7 @@ public class ClusterStatsNodes implements ToXContentFragment {
*/
private OsStats(List<NodeInfo> nodeInfos, List<NodeStats> nodeStatsList) {
this.names = new ObjectIntHashMap<>();
this.prettyNames = new ObjectIntHashMap<>();
int availableProcessors = 0;
int allocatedProcessors = 0;
for (NodeInfo nodeInfo : nodeInfos) {
@ -242,6 +244,9 @@ public class ClusterStatsNodes implements ToXContentFragment {
if (nodeInfo.getOs().getName() != null) {
names.addTo(nodeInfo.getOs().getName(), 1);
}
if (nodeInfo.getOs().getPrettyName() != null) {
prettyNames.addTo(nodeInfo.getOs().getPrettyName(), 1);
}
}
this.availableProcessors = availableProcessors;
this.allocatedProcessors = allocatedProcessors;
@ -280,6 +285,8 @@ public class ClusterStatsNodes implements ToXContentFragment {
static final String ALLOCATED_PROCESSORS = "allocated_processors";
static final String NAME = "name";
static final String NAMES = "names";
static final String PRETTY_NAME = "pretty_name";
static final String PRETTY_NAMES = "pretty_names";
static final String COUNT = "count";
}
@ -289,12 +296,28 @@ public class ClusterStatsNodes implements ToXContentFragment {
builder.field(Fields.AVAILABLE_PROCESSORS, availableProcessors);
builder.field(Fields.ALLOCATED_PROCESSORS, allocatedProcessors);
builder.startArray(Fields.NAMES);
{
for (ObjectIntCursor<String> name : names) {
builder.startObject();
{
builder.field(Fields.NAME, name.key);
builder.field(Fields.COUNT, name.value);
}
builder.endObject();
}
}
builder.endArray();
builder.startArray(Fields.PRETTY_NAMES);
{
for (final ObjectIntCursor<String> prettyName : prettyNames) {
builder.startObject();
{
builder.field(Fields.PRETTY_NAME, prettyName.key);
builder.field(Fields.COUNT, prettyName.value);
}
builder.endObject();
}
}
builder.endArray();
mem.toXContent(builder, params);
return builder;

29
server/src/main/java/org/elasticsearch/monitor/os/DummyOsInfo.java
View File

@ -1,29 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.monitor.os;
public class DummyOsInfo extends OsInfo {
private DummyOsInfo() {
super(0, 0, 0, "dummy_name", "dummy_arch", "dummy_version");
}
public static final DummyOsInfo INSTANCE = new DummyOsInfo();
}

29
server/src/main/java/org/elasticsearch/monitor/os/OsInfo.java
View File

@ -19,11 +19,11 @@
package org.elasticsearch.monitor.os;
import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ToXContent.Params;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -35,14 +35,23 @@ public class OsInfo implements Writeable, ToXContentFragment {
private final int availableProcessors;
private final int allocatedProcessors;
private final String name;
private final String prettyName;
private final String arch;
private final String version;
public OsInfo(long refreshInterval, int availableProcessors, int allocatedProcessors, String name, String arch, String version) {
public OsInfo(
final long refreshInterval,
final int availableProcessors,
final int allocatedProcessors,
final String name,
final String prettyName,
final String arch,
final String version) {
this.refreshInterval = refreshInterval;
this.availableProcessors = availableProcessors;
this.allocatedProcessors = allocatedProcessors;
this.name = name;
this.prettyName = prettyName;
this.arch = arch;
this.version = version;
}
@ -52,6 +61,11 @@ public class OsInfo implements Writeable, ToXContentFragment {
this.availableProcessors = in.readInt();
this.allocatedProcessors = in.readInt();
this.name = in.readOptionalString();
if (in.getVersion().onOrAfter(Version.V_7_0_0)) {
this.prettyName = in.readOptionalString();
} else {
this.prettyName = null;
}
this.arch = in.readOptionalString();
this.version = in.readOptionalString();
}
@ -62,6 +76,9 @@ public class OsInfo implements Writeable, ToXContentFragment {
out.writeInt(availableProcessors);
out.writeInt(allocatedProcessors);
out.writeOptionalString(name);
if (out.getVersion().onOrAfter(Version.V_7_0_0)) {
out.writeOptionalString(prettyName);
}
out.writeOptionalString(arch);
out.writeOptionalString(version);
}
@ -82,6 +99,10 @@ public class OsInfo implements Writeable, ToXContentFragment {
return name;
}
public String getPrettyName() {
return prettyName;
}
public String getArch() {
return arch;
}
@ -93,6 +114,7 @@ public class OsInfo implements Writeable, ToXContentFragment {
static final class Fields {
static final String OS = "os";
static final String NAME = "name";
static final String PRETTY_NAME = "pretty_name";
static final String ARCH = "arch";
static final String VERSION = "version";
static final String REFRESH_INTERVAL = "refresh_interval";
@ -108,6 +130,9 @@ public class OsInfo implements Writeable, ToXContentFragment {
if (name != null) {
builder.field(Fields.NAME, name);
}
if (prettyName != null) {
builder.field(Fields.PRETTY_NAME, prettyName);
}
if (arch != null) {
builder.field(Fields.ARCH, arch);
}

69
server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
View File

@ -19,8 +19,8 @@
package org.elasticsearch.monitor.os;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.util.Constants;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.io.PathUtils;
@ -36,6 +36,8 @@ import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
public class OsProbe {
@ -519,9 +521,68 @@ public class OsProbe {
private final Logger logger = LogManager.getLogger(getClass());
public OsInfo osInfo(long refreshInterval, int allocatedProcessors) {
return new OsInfo(refreshInterval, Runtime.getRuntime().availableProcessors(),
allocatedProcessors, Constants.OS_NAME, Constants.OS_ARCH, Constants.OS_VERSION);
OsInfo osInfo(long refreshInterval, int allocatedProcessors) throws IOException {
return new OsInfo(
refreshInterval,
Runtime.getRuntime().availableProcessors(),
allocatedProcessors,
Constants.OS_NAME,
getPrettyName(),
Constants.OS_ARCH,
Constants.OS_VERSION);
}
private String getPrettyName() throws IOException {
// TODO: return a prettier name on non-Linux OS
if (Constants.LINUX) {
/*
* We read the lines from /etc/os-release (or /usr/lib/os-release) to extract the PRETTY_NAME. The format of this file is
* newline-separated key-value pairs. The key and value are separated by an equals symbol (=). The value can unquoted, or
* wrapped in single- or double-quotes.
*/
final List<String> etcOsReleaseLines = readOsRelease();
final List<String> prettyNameLines =
etcOsReleaseLines.stream().filter(line -> line.startsWith("PRETTY_NAME")).collect(Collectors.toList());
assert prettyNameLines.size() <= 1 : prettyNameLines;
final Optional<String> maybePrettyNameLine =
prettyNameLines.size() == 1 ? Optional.of(prettyNameLines.get(0)) : Optional.empty();
if (maybePrettyNameLine.isPresent()) {
final String prettyNameLine = maybePrettyNameLine.get();
final String[] prettyNameFields = prettyNameLine.split("=");
assert prettyNameFields.length == 2 : prettyNameLine;
if (prettyNameFields[1].length() >= 3 &&
(prettyNameFields[1].startsWith("\"") && prettyNameFields[1].endsWith("\"")) ||
(prettyNameFields[1].startsWith("'") && prettyNameFields[1].endsWith("'"))) {
return prettyNameFields[1].substring(1, prettyNameFields[1].length() - 1);
} else {
return prettyNameFields[1];
}
} else {
return Constants.OS_NAME;
}
} else {
return Constants.OS_NAME;
}
}
/**
* The lines from {@code /etc/os-release} or {@code /usr/lib/os-release} as a fallback. These file represents identification of the
* underlying operating system. The structure of the file is newlines of key-value pairs of shell-compatible variable assignments.
*
* @return the lines from {@code /etc/os-release} or {@code /usr/lib/os-release}
* @throws IOException if an I/O exception occurs reading {@code /etc/os-release} or {@code /usr/lib/os-release}
*/
@SuppressForbidden(reason = "access /etc/os-release or /usr/lib/os-release")
List<String> readOsRelease() throws IOException {
final List<String> lines;
if (Files.exists(PathUtils.get("/etc/os-release"))) {
lines = Files.readAllLines(PathUtils.get("/etc/os-release"));
} else {
lines = Files.readAllLines(PathUtils.get("/usr/lib/os-release"));
}
assert lines != null && lines.isEmpty() == false;
return lines;
}
public OsStats osStats() {

4
server/src/main/java/org/elasticsearch/monitor/os/OsService.java
View File

@ -27,6 +27,8 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.SingleObjectCache;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import java.io.IOException;
public class OsService extends AbstractComponent {
private final OsProbe probe;
@ -37,7 +39,7 @@ public class OsService extends AbstractComponent {
Setting.timeSetting("monitor.os.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1),
Property.NodeScope);
public OsService(Settings settings) {
public OsService(Settings settings) throws IOException {
this.probe = OsProbe.getInstance();
TimeValue refreshInterval = REFRESH_INTERVAL_SETTING.get(settings);
this.info = probe.osInfo(refreshInterval.millis(), EsExecutors.numberOfProcessors(settings));

4
server/src/main/resources/org/elasticsearch/bootstrap/security.policy
View File

@ -124,6 +124,10 @@ grant {
// read max virtual memory areas
permission java.io.FilePermission "/proc/sys/vm/max_map_count", "read";
// OS release on Linux
permission java.io.FilePermission "/etc/os-release", "read";
permission java.io.FilePermission "/usr/lib/os-release", "read";
// io stats on Linux
permission java.io.FilePermission "/proc/self/mountinfo", "read";
permission java.io.FilePermission "/proc/diskstats", "read";

56
server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java
View File

@ -22,8 +22,10 @@ package org.elasticsearch.monitor.os;
import org.apache.lucene.util.Constants;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.math.BigInteger;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.hamcrest.Matchers.allOf;
@ -38,23 +40,53 @@ import static org.hamcrest.Matchers.notNullValue;
public class OsProbeTests extends ESTestCase {
private final OsProbe probe = OsProbe.getInstance();
public void testOsInfo() throws IOException {
final int allocatedProcessors = randomIntBetween(1, Runtime.getRuntime().availableProcessors());
final long refreshInterval = randomBoolean() ? -1 : randomNonNegativeLong();
final String prettyName;
if (Constants.LINUX) {
prettyName = randomFrom("Fedora 28 (Workstation Edition)", "Linux", null);
} else {
prettyName = Constants.OS_NAME;
}
final OsProbe osProbe = new OsProbe() {
public void testOsInfo() {
int allocatedProcessors = randomIntBetween(1, Runtime.getRuntime().availableProcessors());
long refreshInterval = randomBoolean() ? -1 : randomNonNegativeLong();
OsInfo info = probe.osInfo(refreshInterval, allocatedProcessors);
@Override
List<String> readOsRelease() throws IOException {
assert Constants.LINUX : Constants.OS_NAME;
if (prettyName != null) {
final String quote = randomFrom("\"", "'", null);
if (quote == null) {
return Arrays.asList("NAME=" + randomAlphaOfLength(16), "PRETTY_NAME=" + prettyName);
} else {
return Arrays.asList("NAME=" + randomAlphaOfLength(16), "PRETTY_NAME=" + quote + prettyName + quote);
}
} else {
return Collections.singletonList("NAME=" + randomAlphaOfLength(16));
}
}
};
final OsInfo info = osProbe.osInfo(refreshInterval, allocatedProcessors);
assertNotNull(info);
assertEquals(refreshInterval, info.getRefreshInterval());
assertEquals(Constants.OS_NAME, info.getName());
assertEquals(Constants.OS_ARCH, info.getArch());
assertEquals(Constants.OS_VERSION, info.getVersion());
assertEquals(allocatedProcessors, info.getAllocatedProcessors());
assertEquals(Runtime.getRuntime().availableProcessors(), info.getAvailableProcessors());
assertThat(info.getRefreshInterval(), equalTo(refreshInterval));
assertThat(info.getName(), equalTo(Constants.OS_NAME));
if (Constants.LINUX) {
if (prettyName != null) {
assertThat(info.getPrettyName(), equalTo(prettyName));
} else {
assertThat(info.getPrettyName(), equalTo(Constants.OS_NAME));
}
}
assertThat(info.getArch(), equalTo(Constants.OS_ARCH));
assertThat(info.getVersion(), equalTo(Constants.OS_VERSION));
assertThat(info.getAllocatedProcessors(), equalTo(allocatedProcessors));
assertThat(info.getAvailableProcessors(), equalTo(Runtime.getRuntime().availableProcessors()));
}
public void testOsStats() {
OsStats stats = probe.osStats();
final OsProbe osProbe = new OsProbe();
OsStats stats = osProbe.osStats();
assertNotNull(stats);
assertThat(stats.getTimestamp(), greaterThan(0L));
assertThat(stats.getCpu().getPercent(), anyOf(equalTo((short) -1),

2
server/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
View File

@ -118,7 +118,7 @@ public class NodeInfoStreamingTests extends ESTestCase {
String name = randomAlphaOfLengthBetween(3, 10);
String arch = randomAlphaOfLengthBetween(3, 10);
String version = randomAlphaOfLengthBetween(3, 10);
osInfo = new OsInfo(refreshInterval, availableProcessors, allocatedProcessors, name, arch, version);
osInfo = new OsInfo(refreshInterval, availableProcessors, allocatedProcessors, name, name, arch, version);
}
ProcessInfo process = randomBoolean() ? null : new ProcessInfo(randomInt(), randomBoolean(), randomNonNegativeLong());
JvmInfo jvm = randomBoolean() ? null : JvmInfo.jvmInfo();

7
x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
View File

@ -255,6 +255,7 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
when(mockOsInfo.getAvailableProcessors()).thenReturn(32);
when(mockOsInfo.getAllocatedProcessors()).thenReturn(16);
when(mockOsInfo.getName()).thenReturn("_os_name");
when(mockOsInfo.getPrettyName()).thenReturn("_pretty_os_name");
final JvmInfo mockJvmInfo = mock(JvmInfo.class);
when(mockNodeInfo.getJvm()).thenReturn(mockJvmInfo);
@ -446,6 +447,12 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
+ "\"count\":1"
+ "}"
+ "],"
+ "\"pretty_names\":["
+ "{"
+ "\"pretty_name\":\"_pretty_os_name\","
+ "\"count\":1"
+ "}"
+ "],"
+ "\"mem\":{"
+ "\"total_in_bytes\":100,"
+ "\"free_in_bytes\":79,"