From ecce53f0df1c7e63fc4f192bc620c1c416627957 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 17 May 2016 16:16:39 -0400 Subject: [PATCH] Add I/O statistics on Linux This commit adds a variety of real disk metrics for the block devices that back Elasticsearch data paths. A collection of statistics are read from /proc/diskstats and are used to report the raw metrics for operations and read/write bytes. Relates #15915 --- .../org/elasticsearch/env/ESFileStore.java | 29 +- .../elasticsearch/env/NodeEnvironment.java | 8 +- .../elasticsearch/monitor/MonitorService.java | 8 +- .../org/elasticsearch/monitor/fs/FsInfo.java | 251 +++++++++++++++++- .../org/elasticsearch/monitor/fs/FsProbe.java | 82 +++++- .../elasticsearch/monitor/fs/FsService.java | 45 ++-- .../elasticsearch/bootstrap/security.policy | 3 + .../elasticsearch/cluster/DiskUsageTests.java | 12 +- .../monitor/fs/DeviceStatsTests.java | 62 +++++ .../monitor/fs/FsProbeTests.java | 121 ++++++++- docs/reference/cluster/nodes-stats.asciidoc | 49 ++++ .../MockInternalClusterInfoService.java | 2 +- 12 files changed, 624 insertions(+), 48 deletions(-) create mode 100644 core/src/test/java/org/elasticsearch/monitor/fs/DeviceStatsTests.java diff --git a/core/src/main/java/org/elasticsearch/env/ESFileStore.java b/core/src/main/java/org/elasticsearch/env/ESFileStore.java index abd9d2e072f..6df242f7ce1 100644 --- a/core/src/main/java/org/elasticsearch/env/ESFileStore.java +++ b/core/src/main/java/org/elasticsearch/env/ESFileStore.java @@ -32,6 +32,7 @@ import java.nio.file.Path; import java.nio.file.attribute.FileAttributeView; import java.nio.file.attribute.FileStoreAttributeView; import java.util.Arrays; +import java.util.List; /** * Implementation of FileStore that supports @@ -44,6 +45,8 @@ class ESFileStore extends FileStore { final FileStore in; /** Cached result of Lucene's {@code IOUtils.spins} on path. */ final Boolean spins; + int majorDeviceNumber; + int minorDeviceNumber; @SuppressForbidden(reason = "tries to determine if disk is spinning") // TODO: move PathUtils to be package-private here instead of @@ -58,6 +61,21 @@ class ESFileStore extends FileStore { } catch (Exception e) { spins = null; } + try { + final List lines = Files.readAllLines(PathUtils.get("/proc/self/mountinfo")); + for (final String line : lines) { + final String[] fields = line.trim().split("\\s+"); + final String mountPoint = fields[4]; + if (mountPoint.equals(getMountPointLinux(in))) { + final String[] deviceNumbers = fields[2].split(":"); + majorDeviceNumber = Integer.parseInt(deviceNumbers[0]); + minorDeviceNumber = Integer.parseInt(deviceNumbers[1]); + } + } + } catch (Exception e) { + majorDeviceNumber = -1; + minorDeviceNumber = -1; + } } else { spins = null; } @@ -229,10 +247,13 @@ class ESFileStore extends FileStore { @Override public Object getAttribute(String attribute) throws IOException { - if ("lucene:spins".equals(attribute)) { - return spins; - } else { - return in.getAttribute(attribute); + switch(attribute) { + // for the device + case "lucene:spins": return spins; + // for the partition + case "lucene:major_device_number": return majorDeviceNumber; + case "lucene:minor_device_number": return minorDeviceNumber; + default: return in.getAttribute(attribute); } } diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java index c6eec09b1c8..98b47d92246 100644 --- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -64,7 +64,6 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.Semaphore; @@ -88,14 +87,21 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl * not running on Linux, or we hit an exception trying), True means the device possibly spins and False means it does not. */ public final Boolean spins; + public final int majorDeviceNumber; + public final int minorDeviceNumber; + public NodePath(Path path) throws IOException { this.path = path; this.indicesPath = path.resolve(INDICES_FOLDER); this.fileStore = Environment.getFileStore(path); if (fileStore.supportsFileAttributeView("lucene")) { this.spins = (Boolean) fileStore.getAttribute("lucene:spins"); + this.majorDeviceNumber = (int) fileStore.getAttribute("lucene:major_device_number"); + this.minorDeviceNumber = (int) fileStore.getAttribute("lucene:minor_device_number"); } else { this.spins = null; + this.majorDeviceNumber = -1; + this.minorDeviceNumber = -1; } } diff --git a/core/src/main/java/org/elasticsearch/monitor/MonitorService.java b/core/src/main/java/org/elasticsearch/monitor/MonitorService.java index cf033e54d7c..6ae9e2a5b67 100644 --- a/core/src/main/java/org/elasticsearch/monitor/MonitorService.java +++ b/core/src/main/java/org/elasticsearch/monitor/MonitorService.java @@ -31,19 +31,12 @@ import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; -/** - * - */ public class MonitorService extends AbstractLifecycleComponent { private final JvmGcMonitorService jvmGcMonitorService; - private final OsService osService; - private final ProcessService processService; - private final JvmService jvmService; - private final FsService fsService; public MonitorService(Settings settings, NodeEnvironment nodeEnvironment, ThreadPool threadPool) throws IOException { @@ -85,4 +78,5 @@ public class MonitorService extends AbstractLifecycleComponent { protected void doClose() { jvmGcMonitorService.close(); } + } diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java index 1491971cfe2..caa97ea7387 100644 --- a/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java +++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java @@ -188,12 +188,234 @@ public class FsInfo implements Iterable, Writeable, ToXContent { } } + public static class DeviceStats implements Writeable, ToXContent { + + final int majorDeviceNumber; + final int minorDeviceNumber; + final String deviceName; + final long currentReadsCompleted; + final long previousReadsCompleted; + final long currentSectorsRead; + final long previousSectorsRead; + final long currentWritesCompleted; + final long previousWritesCompleted; + final long currentSectorsWritten; + final long previousSectorsWritten; + + public DeviceStats( + final int majorDeviceNumber, + final int minorDeviceNumber, + final String deviceName, + final long currentReadsCompleted, + final long currentSectorsRead, + final long currentWritesCompleted, + final long currentSectorsWritten, + final DeviceStats previousDeviceStats) { + this( + majorDeviceNumber, + minorDeviceNumber, + deviceName, + currentReadsCompleted, + previousDeviceStats != null ? previousDeviceStats.currentReadsCompleted : -1, + currentSectorsWritten, + previousDeviceStats != null ? previousDeviceStats.currentSectorsWritten : -1, + currentSectorsRead, + previousDeviceStats != null ? previousDeviceStats.currentSectorsRead : -1, + currentWritesCompleted, + previousDeviceStats != null ? previousDeviceStats.currentWritesCompleted : -1); + } + + private DeviceStats( + final int majorDeviceNumber, + final int minorDeviceNumber, + final String deviceName, + final long currentReadsCompleted, + final long previousReadsCompleted, + final long currentSectorsWritten, + final long previousSectorsWritten, + final long currentSectorsRead, + final long previousSectorsRead, + final long currentWritesCompleted, + final long previousWritesCompleted) { + this.majorDeviceNumber = majorDeviceNumber; + this.minorDeviceNumber = minorDeviceNumber; + this.deviceName = deviceName; + this.currentReadsCompleted = currentReadsCompleted; + this.previousReadsCompleted = previousReadsCompleted; + this.currentWritesCompleted = currentWritesCompleted; + this.previousWritesCompleted = previousWritesCompleted; + this.currentSectorsRead = currentSectorsRead; + this.previousSectorsRead = previousSectorsRead; + this.currentSectorsWritten = currentSectorsWritten; + this.previousSectorsWritten = previousSectorsWritten; + } + + public DeviceStats(StreamInput in) throws IOException { + majorDeviceNumber = in.readVInt(); + minorDeviceNumber = in.readVInt(); + deviceName = in.readString(); + currentReadsCompleted = in.readLong(); + previousReadsCompleted = in.readLong(); + currentWritesCompleted = in.readLong(); + previousWritesCompleted = in.readLong(); + currentSectorsRead = in.readLong(); + previousSectorsRead = in.readLong(); + currentSectorsWritten = in.readLong(); + previousSectorsWritten = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(majorDeviceNumber); + out.writeVInt(minorDeviceNumber); + out.writeString(deviceName); + out.writeLong(currentReadsCompleted); + out.writeLong(previousReadsCompleted); + out.writeLong(currentWritesCompleted); + out.writeLong(previousWritesCompleted); + out.writeLong(currentSectorsRead); + out.writeLong(previousSectorsRead); + out.writeLong(currentSectorsWritten); + out.writeLong(previousSectorsWritten); + } + + public long operations() { + if (previousReadsCompleted == -1 || previousWritesCompleted == -1) return -1; + + return (currentReadsCompleted - previousReadsCompleted) + + (currentWritesCompleted - previousWritesCompleted); + } + + public long readOperations() { + if (previousReadsCompleted == -1) return -1; + + return (currentReadsCompleted - previousReadsCompleted); + } + + public long writeOperations() { + if (previousWritesCompleted == -1) return -1; + + return (currentWritesCompleted - previousWritesCompleted); + } + + public long readKilobytes() { + if (previousSectorsRead == -1) return -1; + + return (currentSectorsRead - previousSectorsRead) / 2; + } + + public long writeKilobytes() { + if (previousSectorsWritten == -1) return -1; + + return (currentSectorsWritten - previousSectorsWritten) / 2; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("device_name", deviceName); + builder.field(IoStats.OPERATIONS, operations()); + builder.field(IoStats.READ_OPERATIONS, readOperations()); + builder.field(IoStats.WRITE_OPERATIONS, writeOperations()); + builder.field(IoStats.READ_KILOBYTES, readKilobytes()); + builder.field(IoStats.WRITE_KILOBYTES, writeKilobytes()); + return builder; + } + + } + + public static class IoStats implements Writeable, ToXContent { + + private static final String OPERATIONS = "operations"; + private static final String READ_OPERATIONS = "read_operations"; + private static final String WRITE_OPERATIONS = "write_operations"; + private static final String READ_KILOBYTES = "read_kilobytes"; + private static final String WRITE_KILOBYTES = "write_kilobytes"; + + final DeviceStats[] devicesStats; + final long totalOperations; + final long totalReadOperations; + final long totalWriteOperations; + final long totalReadKilobytes; + final long totalWriteKilobytes; + + public IoStats(final DeviceStats[] devicesStats) { + this.devicesStats = devicesStats; + long totalOperations = 0; + long totalReadOperations = 0; + long totalWriteOperations = 0; + long totalReadKilobytes = 0; + long totalWriteKilobytes = 0; + for (DeviceStats deviceStats : devicesStats) { + totalOperations += deviceStats.operations() != -1 ? deviceStats.operations() : 0; + totalReadOperations += deviceStats.readOperations() != -1 ? deviceStats.readOperations() : 0; + totalWriteOperations += deviceStats.writeOperations() != -1 ? deviceStats.writeOperations() : 0; + totalReadKilobytes += deviceStats.readKilobytes() != -1 ? deviceStats.readKilobytes() : 0; + totalWriteKilobytes += deviceStats.writeKilobytes() != -1 ? deviceStats.writeKilobytes() : 0; + } + this.totalOperations = totalOperations; + this.totalReadOperations = totalReadOperations; + this.totalWriteOperations = totalWriteOperations; + this.totalReadKilobytes = totalReadKilobytes; + this.totalWriteKilobytes = totalWriteKilobytes; + } + + public IoStats(StreamInput in) throws IOException { + final int length = in.readVInt(); + final DeviceStats[] devicesStats = new DeviceStats[length]; + for (int i = 0; i < length; i++) { + devicesStats[i] = new DeviceStats(in); + } + this.devicesStats = devicesStats; + this.totalOperations = in.readLong(); + this.totalReadOperations = in.readLong(); + this.totalWriteOperations = in.readLong(); + this.totalReadKilobytes = in.readLong(); + this.totalWriteKilobytes = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(devicesStats.length); + for (int i = 0; i < devicesStats.length; i++) { + devicesStats[i].writeTo(out); + } + out.writeLong(totalOperations); + out.writeLong(totalReadOperations); + out.writeLong(totalWriteOperations); + out.writeLong(totalReadKilobytes); + out.writeLong(totalWriteKilobytes); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (devicesStats.length > 0) { + builder.startArray("devices"); + for (DeviceStats deviceStats : devicesStats) { + builder.startObject(); + deviceStats.toXContent(builder, params); + builder.endObject(); + } + builder.endArray(); + builder.startObject("total"); + builder.field(OPERATIONS, totalOperations); + builder.field(READ_OPERATIONS, totalReadOperations); + builder.field(WRITE_OPERATIONS, totalWriteOperations); + builder.field(READ_KILOBYTES, totalReadKilobytes); + builder.field(WRITE_KILOBYTES, totalWriteKilobytes); + } + return builder; + } + + } + final long timestamp; final Path[] paths; + final IoStats ioStats; Path total; - public FsInfo(long timestamp, Path[] paths) { + public FsInfo(long timestamp, IoStats ioStats, Path[] paths) { this.timestamp = timestamp; + this.ioStats = ioStats; this.paths = paths; this.total = null; } @@ -203,6 +425,7 @@ public class FsInfo implements Iterable, Writeable, ToXContent { */ public FsInfo(StreamInput in) throws IOException { timestamp = in.readVLong(); + ioStats = in.readOptionalWriteable(IoStats::new); paths = new Path[in.readVInt()]; for (int i = 0; i < paths.length; i++) { paths[i] = new Path(in); @@ -212,6 +435,7 @@ public class FsInfo implements Iterable, Writeable, ToXContent { @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(timestamp); + out.writeOptionalWriteable(ioStats); out.writeVInt(paths.length); for (Path path : paths) { path.writeTo(out); @@ -244,18 +468,15 @@ public class FsInfo implements Iterable, Writeable, ToXContent { return timestamp; } + public IoStats getIoStats() { + return ioStats; + } + @Override public Iterator iterator() { return Arrays.stream(paths).iterator(); } - static final class Fields { - static final String FS = "fs"; - static final String TIMESTAMP = "timestamp"; - static final String DATA = "data"; - static final String TOTAL = "total"; - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.FS); @@ -267,7 +488,21 @@ public class FsInfo implements Iterable, Writeable, ToXContent { path.toXContent(builder, params); } builder.endArray(); + if (ioStats != null) { + builder.startObject(Fields.IO_STATS); + ioStats.toXContent(builder, params); + builder.endObject(); + } builder.endObject(); return builder; } + + static final class Fields { + static final String FS = "fs"; + static final String TIMESTAMP = "timestamp"; + static final String DATA = "data"; + static final String TOTAL = "total"; + static final String IO_STATS = "io_stats"; + } + } diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java index dc1958f666b..f0b61556924 100644 --- a/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java +++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java @@ -19,12 +19,23 @@ package org.elasticsearch.monitor.fs; +import org.apache.lucene.util.Constants; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.NodeEnvironment.NodePath; import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; public class FsProbe extends AbstractComponent { @@ -35,16 +46,80 @@ public class FsProbe extends AbstractComponent { this.nodeEnv = nodeEnv; } - public FsInfo stats() throws IOException { + public FsInfo stats(FsInfo previous) throws IOException { if (!nodeEnv.hasNodeFile()) { - return new FsInfo(System.currentTimeMillis(), new FsInfo.Path[0]); + return new FsInfo(System.currentTimeMillis(), null, new FsInfo.Path[0]); } NodePath[] dataLocations = nodeEnv.nodePaths(); FsInfo.Path[] paths = new FsInfo.Path[dataLocations.length]; for (int i = 0; i < dataLocations.length; i++) { paths[i] = getFSInfo(dataLocations[i]); } - return new FsInfo(System.currentTimeMillis(), paths); + FsInfo.IoStats ioStats = null; + if (Constants.LINUX) { + Set> devicesNumbers = new HashSet<>(); + for (int i = 0; i < dataLocations.length; i++) { + if (dataLocations[i].majorDeviceNumber != -1 && dataLocations[i].minorDeviceNumber != -1) { + devicesNumbers.add(Tuple.tuple(dataLocations[i].majorDeviceNumber, dataLocations[i].minorDeviceNumber)); + } + } + ioStats = ioStats(devicesNumbers, previous); + } + return new FsInfo(System.currentTimeMillis(), ioStats, paths); + } + + final FsInfo.IoStats ioStats(final Set> devicesNumbers, final FsInfo previous) { + try { + final Map, FsInfo.DeviceStats> deviceMap = new HashMap<>(); + if (previous != null && previous.getIoStats() != null && previous.getIoStats().devicesStats != null) { + for (int i = 0; i < previous.getIoStats().devicesStats.length; i++) { + FsInfo.DeviceStats deviceStats = previous.getIoStats().devicesStats[i]; + deviceMap.put(Tuple.tuple(deviceStats.majorDeviceNumber, deviceStats.minorDeviceNumber), deviceStats); + } + } + + List devicesStats = new ArrayList<>(); + + List lines = readProcDiskStats(); + if (!lines.isEmpty()) { + for (String line : lines) { + String fields[] = line.trim().split("\\s+"); + final int majorDeviceNumber = Integer.parseInt(fields[0]); + final int minorDeviceNumber = Integer.parseInt(fields[1]); + if (!devicesNumbers.contains(Tuple.tuple(majorDeviceNumber, minorDeviceNumber))) { + continue; + } + final String deviceName = fields[2]; + final long readsCompleted = Long.parseLong(fields[3]); + final long sectorsRead = Long.parseLong(fields[5]); + final long writesCompleted = Long.parseLong(fields[7]); + final long sectorsWritten = Long.parseLong(fields[9]); + final FsInfo.DeviceStats deviceStats = + new FsInfo.DeviceStats( + majorDeviceNumber, + minorDeviceNumber, + deviceName, + readsCompleted, + sectorsRead, + writesCompleted, + sectorsWritten, + deviceMap.get(Tuple.tuple(majorDeviceNumber, minorDeviceNumber))); + devicesStats.add(deviceStats); + } + } + + return new FsInfo.IoStats(devicesStats.toArray(new FsInfo.DeviceStats[devicesStats.size()])); + } catch (Exception e) { + // do not fail Elasticsearch if something unexpected + // happens here + logger.debug("unexpected exception processing /proc/diskstats for devices {}", e, devicesNumbers); + return null; + } + } + + @SuppressForbidden(reason = "read /proc/diskstats") + List readProcDiskStats() throws IOException { + return Files.readAllLines(PathUtils.get("/proc/diskstats")); } public static FsInfo.Path getFSInfo(NodePath nodePath) throws IOException { @@ -62,4 +137,5 @@ public class FsProbe extends AbstractComponent { fsPath.spins = nodePath.spins; return fsPath; } + } diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java index d1058652d34..e04a06fc460 100644 --- a/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java +++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java @@ -20,6 +20,7 @@ package org.elasticsearch.monitor.fs; import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; @@ -29,44 +30,54 @@ import org.elasticsearch.env.NodeEnvironment; import java.io.IOException; -/** - */ public class FsService extends AbstractComponent { private final FsProbe probe; - - private final SingleObjectCache fsStatsCache; + private final TimeValue refreshInterval; + private final SingleObjectCache cache; public final static Setting REFRESH_INTERVAL_SETTING = - Setting.timeSetting("monitor.fs.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1), + Setting.timeSetting( + "monitor.fs.refresh_interval", + TimeValue.timeValueSeconds(1), + TimeValue.timeValueSeconds(1), Property.NodeScope); - public FsService(Settings settings, NodeEnvironment nodeEnvironment) throws IOException { + public FsService(final Settings settings, final NodeEnvironment nodeEnvironment) { super(settings); this.probe = new FsProbe(settings, nodeEnvironment); - TimeValue refreshInterval = REFRESH_INTERVAL_SETTING.get(settings); - fsStatsCache = new FsInfoCache(refreshInterval, probe.stats()); + refreshInterval = REFRESH_INTERVAL_SETTING.get(settings); logger.debug("using refresh_interval [{}]", refreshInterval); + cache = new FsInfoCache(refreshInterval, stats(probe, null, logger)); } public FsInfo stats() { - return fsStatsCache.getOrRefresh(); + return cache.getOrRefresh(); + } + + private static FsInfo stats(FsProbe probe, FsInfo initialValue, ESLogger logger) { + try { + return probe.stats(initialValue); + } catch (IOException e) { + logger.debug("unexpected exception reading filesystem info", e); + return null; + } } private class FsInfoCache extends SingleObjectCache { - public FsInfoCache(TimeValue interval, FsInfo initValue) { - super(interval, initValue); + + private final FsInfo initialValue; + + public FsInfoCache(TimeValue interval, FsInfo initialValue) { + super(interval, initialValue); + this.initialValue = initialValue; } @Override protected FsInfo refresh() { - try { - return probe.stats(); - } catch (IOException ex) { - logger.warn("Failed to fetch fs stats - returning empty instance"); - return new FsInfo(0, null); - } + return stats(probe, initialValue, logger); } + } } diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy index 8919716bfb1..74404903e91 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -124,4 +124,7 @@ grant { // read max virtual memory areas permission java.io.FilePermission "/proc/sys/vm/max_map_count", "read"; + + // io stats on Linux + permission java.io.FilePermission "/proc/diskstats", "read"; }; diff --git a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java index d7738b1abcb..9210e2a56bd 100644 --- a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java @@ -201,11 +201,11 @@ public class DiskUsageTests extends ESTestCase { }; List nodeStats = Arrays.asList( new NodeStats(new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null,new FsInfo(0, node1FSInfo), null,null,null,null,null, null), + null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null), new NodeStats(new DiscoveryNode("node_2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, node2FSInfo), null,null,null,null,null, null), + null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null), new NodeStats(new DiscoveryNode("node_3", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, node3FSInfo), null,null,null,null,null, null) + null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null) ); InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvaiableUsages, newMostAvaiableUsages); DiskUsage leastNode_1 = newLeastAvaiableUsages.get("node_1"); @@ -242,11 +242,11 @@ public class DiskUsageTests extends ESTestCase { }; List nodeStats = Arrays.asList( new NodeStats(new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null,new FsInfo(0, node1FSInfo), null,null,null,null,null, null), + null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null), new NodeStats(new DiscoveryNode("node_2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, node2FSInfo), null,null,null,null,null, null), + null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null), new NodeStats(new DiscoveryNode("node_3", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, node3FSInfo), null,null,null,null,null, null) + null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null) ); InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvailableUsages, newMostAvailableUsages); DiskUsage leastNode_1 = newLeastAvailableUsages.get("node_1"); diff --git a/core/src/test/java/org/elasticsearch/monitor/fs/DeviceStatsTests.java b/core/src/test/java/org/elasticsearch/monitor/fs/DeviceStatsTests.java new file mode 100644 index 00000000000..758f67ff563 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/monitor/fs/DeviceStatsTests.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.monitor.fs; + +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.CoreMatchers.equalTo; + +public class DeviceStatsTests extends ESTestCase { + + public void testDeviceStats() { + final int majorDeviceNumber = randomIntBetween(1, 1 << 8); + final int minorDeviceNumber = randomIntBetween(0, 1 << 5); + final String deviceName = randomAsciiOfLength(3); + final int readsCompleted = randomIntBetween(1, 1 << 16); + final int sectorsRead = randomIntBetween(8 * readsCompleted, 16 * readsCompleted); + final int writesCompleted = randomIntBetween(1, 1 << 16); + final int sectorsWritten = randomIntBetween(8 * writesCompleted, 16 * writesCompleted); + + FsInfo.DeviceStats previous = new FsInfo.DeviceStats( + majorDeviceNumber, + minorDeviceNumber, + deviceName, + readsCompleted, + sectorsRead, + writesCompleted, + sectorsWritten, + null); + FsInfo.DeviceStats current = new FsInfo.DeviceStats( + majorDeviceNumber, + minorDeviceNumber, + deviceName, + readsCompleted + 1024, + sectorsRead + 16384, + writesCompleted + 2048, + sectorsWritten + 32768, + previous); + assertThat(current.operations(), equalTo(1024L + 2048L)); + assertThat(current.readOperations(), equalTo(1024L)); + assertThat(current.writeOperations(), equalTo(2048L)); + assertThat(current.readKilobytes(), equalTo(16384L / 2)); + assertThat(current.writeKilobytes(), equalTo(32768L / 2)); + } + +} diff --git a/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java b/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java index 26a56529e05..8eba98ae0b7 100644 --- a/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java +++ b/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java @@ -19,25 +19,55 @@ package org.elasticsearch.monitor.fs; +import org.apache.lucene.util.Constants; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.isEmptyOrNullString; import static org.hamcrest.Matchers.not; public class FsProbeTests extends ESTestCase { + public void testFsInfo() throws IOException { + try (NodeEnvironment env = newNodeEnvironment()) { FsProbe probe = new FsProbe(Settings.EMPTY, env); - FsInfo stats = probe.stats(); + FsInfo stats = probe.stats(null); assertNotNull(stats); assertThat(stats.getTimestamp(), greaterThan(0L)); + if (Constants.LINUX) { + assertNotNull(stats.getIoStats()); + assertNotNull(stats.getIoStats().devicesStats); + for (int i = 0; i < stats.getIoStats().devicesStats.length; i++) { + final FsInfo.DeviceStats deviceStats = stats.getIoStats().devicesStats[i]; + assertNotNull(deviceStats); + assertThat(deviceStats.currentReadsCompleted, greaterThanOrEqualTo(0L)); + assertThat(deviceStats.previousReadsCompleted, equalTo(-1L)); + assertThat(deviceStats.currentSectorsRead, greaterThanOrEqualTo(0L)); + assertThat(deviceStats.previousSectorsRead, equalTo(-1L)); + assertThat(deviceStats.currentWritesCompleted, greaterThanOrEqualTo(0L)); + assertThat(deviceStats.previousWritesCompleted, equalTo(-1L)); + assertThat(deviceStats.currentSectorsWritten, greaterThanOrEqualTo(0L)); + assertThat(deviceStats.previousSectorsWritten, equalTo(-1L)); + } + } else { + assertNull(stats.getIoStats()); + } + FsInfo.Path total = stats.getTotal(); assertNotNull(total); assertThat(total.total, greaterThan(0L)); @@ -55,4 +85,93 @@ public class FsProbeTests extends ESTestCase { } } } + + public void testIoStats() { + final AtomicReference> diskStats = new AtomicReference<>(); + diskStats.set(Arrays.asList( + " 259 0 nvme0n1 336609 0 7923613 82813 10264051 0 182983933 52451441 0 2970886 52536260", + " 259 1 nvme0n1p1 602 0 9919 131 1 0 1 0 0 19 131", + " 259 2 nvme0n1p2 186 0 8626 18 24 0 60 20 0 34 38", + " 259 3 nvme0n1p3 335733 0 7901620 82658 9592875 0 182983872 50843431 0 1737726 50926087", + " 253 0 dm-0 287716 0 7184666 33457 8398869 0 118857776 18730966 0 1918440 18767169", + " 253 1 dm-1 112 0 4624 13 0 0 0 0 0 5 13", + " 253 2 dm-2 47802 0 710658 49312 1371977 0 64126096 33730596 0 1058193 33781827")); + + final FsProbe probe = new FsProbe(Settings.EMPTY, null) { + @Override + List readProcDiskStats() throws IOException { + return diskStats.get(); + } + }; + + final Set> devicesNumbers = new HashSet<>(); + devicesNumbers.add(Tuple.tuple(253, 0)); + devicesNumbers.add(Tuple.tuple(253, 2)); + final FsInfo.IoStats first = probe.ioStats(devicesNumbers, null); + assertNotNull(first); + assertThat(first.devicesStats[0].majorDeviceNumber, equalTo(253)); + assertThat(first.devicesStats[0].minorDeviceNumber, equalTo(0)); + assertThat(first.devicesStats[0].deviceName, equalTo("dm-0")); + assertThat(first.devicesStats[0].currentReadsCompleted, equalTo(287716L)); + assertThat(first.devicesStats[0].previousReadsCompleted, equalTo(-1L)); + assertThat(first.devicesStats[0].currentSectorsRead, equalTo(7184666L)); + assertThat(first.devicesStats[0].previousSectorsRead, equalTo(-1L)); + assertThat(first.devicesStats[0].currentWritesCompleted, equalTo(8398869L)); + assertThat(first.devicesStats[0].previousWritesCompleted, equalTo(-1L)); + assertThat(first.devicesStats[0].currentSectorsWritten, equalTo(118857776L)); + assertThat(first.devicesStats[0].previousSectorsWritten, equalTo(-1L)); + assertThat(first.devicesStats[1].majorDeviceNumber, equalTo(253)); + assertThat(first.devicesStats[1].minorDeviceNumber, equalTo(2)); + assertThat(first.devicesStats[1].deviceName, equalTo("dm-2")); + assertThat(first.devicesStats[1].currentReadsCompleted, equalTo(47802L)); + assertThat(first.devicesStats[1].previousReadsCompleted, equalTo(-1L)); + assertThat(first.devicesStats[1].currentSectorsRead, equalTo(710658L)); + assertThat(first.devicesStats[1].previousSectorsRead, equalTo(-1L)); + assertThat(first.devicesStats[1].currentWritesCompleted, equalTo(1371977L)); + assertThat(first.devicesStats[1].previousWritesCompleted, equalTo(-1L)); + assertThat(first.devicesStats[1].currentSectorsWritten, equalTo(64126096L)); + assertThat(first.devicesStats[1].previousSectorsWritten, equalTo(-1L)); + + diskStats.set(Arrays.asList( + " 259 0 nvme0n1 336870 0 7928397 82876 10264393 0 182986405 52451610 0 2971042 52536492", + " 259 1 nvme0n1p1 602 0 9919 131 1 0 1 0 0 19 131", + " 259 2 nvme0n1p2 186 0 8626 18 24 0 60 20 0 34 38", + " 259 3 nvme0n1p3 335994 0 7906404 82721 9593184 0 182986344 50843529 0 1737840 50926248", + " 253 0 dm-0 287734 0 7185242 33464 8398869 0 118857776 18730966 0 1918444 18767176", + " 253 1 dm-1 112 0 4624 13 0 0 0 0 0 5 13", + " 253 2 dm-2 48045 0 714866 49369 1372291 0 64128568 33730766 0 1058347 33782056")); + + final FsInfo previous = new FsInfo(System.currentTimeMillis(), first, null); + final FsInfo.IoStats second = probe.ioStats(devicesNumbers, previous); + assertNotNull(second); + assertThat(second.devicesStats[0].majorDeviceNumber, equalTo(253)); + assertThat(second.devicesStats[0].minorDeviceNumber, equalTo(0)); + assertThat(second.devicesStats[0].deviceName, equalTo("dm-0")); + assertThat(second.devicesStats[0].currentReadsCompleted, equalTo(287734L)); + assertThat(second.devicesStats[0].previousReadsCompleted, equalTo(287716L)); + assertThat(second.devicesStats[0].currentSectorsRead, equalTo(7185242L)); + assertThat(second.devicesStats[0].previousSectorsRead, equalTo(7184666L)); + assertThat(second.devicesStats[0].currentWritesCompleted, equalTo(8398869L)); + assertThat(second.devicesStats[0].previousWritesCompleted, equalTo(8398869L)); + assertThat(second.devicesStats[0].currentSectorsWritten, equalTo(118857776L)); + assertThat(second.devicesStats[0].previousSectorsWritten, equalTo(118857776L)); + assertThat(second.devicesStats[1].majorDeviceNumber, equalTo(253)); + assertThat(second.devicesStats[1].minorDeviceNumber, equalTo(2)); + assertThat(second.devicesStats[1].deviceName, equalTo("dm-2")); + assertThat(second.devicesStats[1].currentReadsCompleted, equalTo(48045L)); + assertThat(second.devicesStats[1].previousReadsCompleted, equalTo(47802L)); + assertThat(second.devicesStats[1].currentSectorsRead, equalTo(714866L)); + assertThat(second.devicesStats[1].previousSectorsRead, equalTo(710658L)); + assertThat(second.devicesStats[1].currentWritesCompleted, equalTo(1372291L)); + assertThat(second.devicesStats[1].previousWritesCompleted, equalTo(1371977L)); + assertThat(second.devicesStats[1].currentSectorsWritten, equalTo(64128568L)); + assertThat(second.devicesStats[1].previousSectorsWritten, equalTo(64126096L)); + + assertThat(second.totalOperations, equalTo(575L)); + assertThat(second.totalReadOperations, equalTo(261L)); + assertThat(second.totalWriteOperations, equalTo(314L)); + assertThat(second.totalReadKilobytes, equalTo(2392L)); + assertThat(second.totalWriteKilobytes, equalTo(1236L)); + } + } diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc index 8a2f27112b5..e49e2f6b8aa 100644 --- a/docs/reference/cluster/nodes-stats.asciidoc +++ b/docs/reference/cluster/nodes-stats.asciidoc @@ -121,6 +121,55 @@ information that concern the file system: `null` means we could not determine it, `true` means the device possibly spins and `false` means it does not (ex: solid-state disks). +`fs.io_stats.devices` (Linux only):: + Array of disk metrics for each device that is backing an + Elasticsearch data path. These disk metrics are probed periodically + and averages between the last probe and the current probe are + computed. + +`fs.io_stats.devices.device_name` (Linux only):: + The Linux device name. + +`fs.io_stats.devices.operations` (Linux only):: + The total number of read and write operations for the device + completed since starting Elasticsearch. + +`fs.io_stats.devices.read_operations` (Linux only):: + The total number of read operations for the device completed since + starting Elasticsearch. + +`fs.io_stats.devices.write_operations` (Linux only):: + The total number of write operations for the device completed since + starting Elasticsearch. + +`fs.io_stats.devices.read_kilobytes` (Linux only):: + The total number of kilobytes read for the device since starting + Elasticsearch. + +`fs.io_stats.devices.write_kilobytes` (Linux only):: + The total number of kilobytes written for the device since + starting Elasticsearch. + +`fs.io_stats.operations` (Linux only):: + The total number of read and write operations across all devices + used by Elasticsearch completed since starting Elasticsearch. + +`fs.io_stats.read_operations` (Linux only):: + The total number of read operations for across all devices used by + Elasticsearch completed since starting Elasticsearch. + +`fs.io_stats.write_operations` (Linux only):: + The total number of write operations across all devices used by + Elasticsearch completed since starting Elasticsearch. + +`fs.io_stats.read_kilobytes` (Linux only):: + The total number of kilobytes read across all devices used by + Elasticsearch since starting Elasticsearch. + +`fs.io_stats.write_kilobytes` (Linux only):: + The total number of kilobytes written across all devices used by + Elasticsearch since starting Elasticsearch. + [float] [[os-stats]] ==== Operating System statistics diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java index ef8857475b3..cf881f4cd66 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java @@ -74,7 +74,7 @@ public class MockInternalClusterInfoService extends InternalClusterInfoService { FsInfo.Path path = new FsInfo.Path("/dev/null", null, usage.getTotalBytes(), usage.getFreeBytes(), usage.getFreeBytes()); paths[0] = path; - FsInfo fsInfo = new FsInfo(System.currentTimeMillis(), paths); + FsInfo fsInfo = new FsInfo(System.currentTimeMillis(), null, paths); return new NodeStats(new DiscoveryNode(nodeName, DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), System.currentTimeMillis(), null, null, null, null, null,