honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into feature/ingest

This commit is contained in:
Tal Levy 2015-11-17 12:08:26 -08:00
parent ef4d2a9426 185027a0ff
commit d0b5fc8bb4
33 changed files with 343 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
build.gradle
View File

@ -171,5 +171,9 @@ task clean(type: GradleBuild) {
tasks = ['clean']
}
task run(dependsOn: ':distribution:run')
task run() {
dependsOn ':distribution:run'
description = 'Runs elasticsearch in the foreground'
group = 'Verification'
}

154
buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy
View File

@ -23,45 +23,163 @@ import com.carrotsearch.ant.tasks.junit4.JUnit4
import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.eventbus.Subscribe
import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedStartEvent
import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedSuiteResultEvent
import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedTestResultEvent
import com.carrotsearch.ant.tasks.junit4.listeners.AggregatedEventListener
import org.gradle.logging.ProgressLogger
import org.gradle.logging.ProgressLoggerFactory
import org.junit.runner.Description
import java.util.concurrent.atomic.AtomicInteger
import static com.carrotsearch.ant.tasks.junit4.events.aggregated.TestStatus.*
import static com.carrotsearch.ant.tasks.junit4.FormattingUtils.formatDurationInSeconds
import static java.lang.Math.max
/**
* Adapts junit4's event listeners into gradle's ProgressLogger. Note that
* junit4 guarantees (via guava) that methods on this class won't be called by
* multiple threads simultaneously which is helpful in making it simpler.
*
* Every time a test finishes this class will update the logger. It will log
* the last finished test method on the logger line until the first suite
* finishes. Once the first suite finishes it always logs the last finished
* suite. This means that in test runs with a single suite the logger will be
* updated with the test name the whole time which is useful because these runs
* usually have longer individual tests. For test runs with lots of suites the
* majority of the time is spent showing the last suite that finished which is
* more useful for those test runs because test methods there tend to be very
* quick.
*/
class TestProgressLogger implements AggregatedEventListener {
/** Factory to build a progress logger when testing starts */
ProgressLoggerFactory factory
ProgressLogger progressLogger
int totalSuites;
AtomicInteger suitesCompleted = new AtomicInteger()
AtomicInteger testsCompleted = new AtomicInteger()
AtomicInteger testsFailed = new AtomicInteger()
AtomicInteger testsIgnored = new AtomicInteger()
int totalSuites
int totalSlaves
// sprintf formats used to align the integers we print
String suitesFormat
String slavesFormat
String testsFormat
// Counters incremented test completion.
volatile int suitesCompleted = 0
volatile int testsCompleted = 0
volatile int testsFailed = 0
volatile int testsIgnored = 0
// Information about the last, most interesting event.
volatile String eventDescription
volatile int eventSlave
volatile long eventExecutionTime
/** Have we finished a whole suite yet? */
volatile boolean suiteFinished = false
/* Note that we probably overuse volatile here but it isn't hurting us and
lets us move things around without worying about breaking things. */
@Subscribe
void onStart(AggregatedStartEvent e) throws IOException {
totalSuites = e.getSuiteCount();
totalSuites = e.suiteCount
totalSlaves = e.slaveCount
progressLogger = factory.newOperation(TestProgressLogger)
progressLogger.setDescription('Randomized test runner')
progressLogger.started()
progressLogger.progress('Starting JUnit4 with ' + e.getSlaveCount() + ' jvms')
progressLogger.progress(
"Starting JUnit4 for ${totalSuites} suites on ${totalSlaves} jvms")
suitesFormat = "%0${widthForTotal(totalSuites)}d"
slavesFormat = "%-${widthForTotal(totalSlaves)}s"
/* Just guess the number of tests because we can't figure it out from
here and it isn't worth doing anything fancy to prevent the console
from jumping around a little. 200 is a pretty wild guess for the
minimum but it makes REST tests output sanely. */
int totalNumberOfTestsGuess = max(200, totalSuites * 10)
testsFormat = "%0${widthForTotal(totalNumberOfTestsGuess)}d"
}
@Subscribe
void onTestResult(AggregatedTestResultEvent e) throws IOException {
testsCompleted++
switch (e.status) {
case ERROR:
case FAILURE:
testsFailed++
break
case IGNORED:
case IGNORED_ASSUMPTION:
testsIgnored++
break
case OK:
break
default:
throw new IllegalArgumentException(
"Unknown test status: [${e.status}]")
}
if (!suiteFinished) {
updateEventInfo(e)
}
log()
}
@Subscribe
void onSuiteResult(AggregatedSuiteResultEvent e) throws IOException {
final int suitesCompleted = suitesCompleted.incrementAndGet();
final int testsCompleted = testsCompleted.addAndGet(e.getDescription().testCount())
final int testsFailed = testsFailed.addAndGet(e.getErrorCount() + e.getFailureCount())
final int testsIgnored = testsIgnored.addAndGet(e.getIgnoredCount())
Description description = e.getDescription()
String suiteName = description.getDisplayName();
suiteName = suiteName.substring(suiteName.lastIndexOf('.') + 1);
progressLogger.progress('Suites [' + suitesCompleted + '/' + totalSuites + '], Tests [' + testsCompleted + '|' + testsFailed + '|' + testsIgnored + '], ' + suiteName + ' on J' + e.getSlave().id + ' in ' + formatDurationInSeconds(e.getExecutionTime()))
suitesCompleted++
suiteFinished = true
updateEventInfo(e)
log()
}
/**
* Update the suite information with a junit4 event.
*/
private void updateEventInfo(Object e) {
eventDescription = simpleName(e.description.className)
if (e.description.methodName != null) {
eventDescription += "#${e.description.methodName}"
}
eventSlave = e.slave.id
eventExecutionTime = e.executionTime
}
/**
* Extract a Class#getSimpleName style name from Class#getName style
* string. We can't just use Class#getSimpleName because junit descriptions
* don't alway s set the class field but they always set the className
* field.
*/
private static String simpleName(String className) {
return className.substring(className.lastIndexOf('.') + 1)
}
private void log() {
/* Remember that instances of this class are only ever active on one
thread at a time so there really aren't race conditions here. It'd be
OK if there were because they'd only display an overcount
temporarily. */
String log = ''
if (totalSuites > 1) {
/* Skip printing the suites to save space when there is only a
single suite. This is nice because when there is only a single
suite we log the method name and those can be long. */
log += sprintf("Suites [${suitesFormat}/${suitesFormat}], ",
[suitesCompleted, totalSuites])
}
log += sprintf("Tests [${testsFormat}|%d|%d], ",
[testsCompleted, testsFailed, testsIgnored])
log += "in ${formatDurationInSeconds(eventExecutionTime)} "
if (totalSlaves > 1) {
/* Skip printing the slaves if there is only one of them. This is
nice because when there is only a single slave there is often
only a single suite and we could use the extra space to log the
test method names. */
log += "J${sprintf(slavesFormat, eventSlave)} "
}
log += "completed ${eventDescription}"
progressLogger.progress(log)
}
private static int widthForTotal(int total) {
return ((total - 1) as String).length()
}
@Override

4
buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View File

@ -240,6 +240,10 @@ class BuildPlugin implements Plugin<Project> {
'X-Compile-Elasticsearch-Version': VersionProperties.elasticsearch,
'X-Compile-Lucene-Version': VersionProperties.lucene,
'Build-Date': ZonedDateTime.now(ZoneOffset.UTC))
if (jarTask.manifest.attributes.containsKey('Change') == false) {
logger.warn('Building without git revision id.')
jarTask.manifest.attributes('Change': 'N/A')
}
}
}
}

25
buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
View File

@ -18,9 +18,9 @@
*/
package org.elasticsearch.gradle.plugin
import nebula.plugin.extraconfigurations.ProvidedBasePlugin
import org.elasticsearch.gradle.BuildPlugin
import org.elasticsearch.gradle.test.RestIntegTestTask
import org.elasticsearch.gradle.test.RunTask
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.tasks.bundling.Zip
@ -33,26 +33,21 @@ class PluginBuildPlugin extends BuildPlugin {
@Override
void apply(Project project) {
super.apply(project)
// TODO: add target compatibility (java version) to elasticsearch properties and set for the project
configureDependencies(project)
// this afterEvaluate must happen before the afterEvaluate added by integTest configure,
// so that the file name resolution for installing the plugin will be setup
project.afterEvaluate {
project.jar.configure {
baseName project.pluginProperties.extension.name
}
project.bundlePlugin.configure {
baseName project.pluginProperties.extension.name
}
project.integTest.configure {
dependsOn project.bundlePlugin
cluster {
plugin project.pluginProperties.extension.name, project.bundlePlugin.outputs.files
}
}
String name = project.pluginProperties.extension.name
project.jar.baseName = name
project.bundlePlugin.baseName = name
project.integTest.dependsOn(project.bundlePlugin)
project.integTest.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
project.tasks.run.dependsOn(project.bundlePlugin)
project.tasks.run.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
}
Task bundle = configureBundleTask(project)
RestIntegTestTask.configure(project)
RunTask.configure(project)
Task bundle = configureBundleTask(project)
project.configurations.archives.artifacts.removeAll { it.archiveTask.is project.jar }
project.configurations.getByName('default').extendsFrom = []
project.artifacts {

9
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
View File

@ -102,7 +102,7 @@ class ClusterFormationTasks {
String camelName = plugin.getKey().replaceAll(/-(\w)/) { _, c -> c.toUpperCase(Locale.ROOT) }
String taskName = "${task.name}#install${camelName[0].toUpperCase(Locale.ROOT) + camelName.substring(1)}Plugin"
// delay reading the file location until execution time by wrapping in a closure within a GString
String file = "${ -> new File(pluginsTmpDir, plugin.getValue().singleFile.getName()).toURI().toURL().toString() }"
String file = "${-> new File(pluginsTmpDir, plugin.getValue().singleFile.getName()).toURI().toURL().toString()}"
Object[] args = [new File(home, 'bin/plugin'), 'install', file]
setup = configureExecTask(taskName, project, setup, cwd, args)
}
@ -115,8 +115,11 @@ class ClusterFormationTasks {
Task start = configureStartTask("${task.name}#start", project, setup, cwd, config, clusterName, pidFile, home)
task.dependsOn(start)
Task stop = configureStopTask("${task.name}#stop", project, [], pidFile)
task.finalizedBy(stop)
if (config.daemonize) {
// if we are running in the background, make sure to stop the server when the task completes
Task stop = configureStopTask("${task.name}#stop", project, [], pidFile)
task.finalizedBy(stop)
}
}
/** Adds a task to extract the elasticsearch distribution */

15
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy
View File

@ -1,12 +1,23 @@
package org.elasticsearch.gradle.test
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.TaskAction
import org.gradle.api.Project
class RunTask extends DefaultTask {
ClusterConfiguration clusterConfig = new ClusterConfiguration(httpPort: 9200, transportPort: 9300, daemonize: false)
RunTask() {
ClusterFormationTasks.setup(project, this, clusterConfig)
project.afterEvaluate {
ClusterFormationTasks.setup(project, this, clusterConfig)
}
}
static void configure(Project project) {
RunTask task = project.tasks.create(
name: 'run',
type: RunTask,
description: "Runs elasticsearch with '${project.path}'",
group: 'Verification')
}
}

14
core/src/main/java/org/elasticsearch/Build.java
View File

@ -33,9 +33,13 @@ import java.util.jar.JarInputStream;
import java.util.jar.Manifest;
/**
* Information about a build of Elasticsearch.
*/
public class Build {
/**
* The current build of Elasticsearch. Filled with information scanned at
* startup from the jar.
*/
public static final Build CURRENT;
static {
@ -56,6 +60,14 @@ public class Build {
shortHash = "Unknown";
date = "Unknown";
}
if (shortHash == null) {
throw new IllegalStateException("Error finding the build shortHash. " +
"Stopping Elasticsearch now so it doesn't run in subtly broken ways. This is likely a build bug.");
}
if (date == null) {
throw new IllegalStateException("Error finding the build date. " +
"Stopping Elasticsearch now so it doesn't run in subtly broken ways. This is likely a build bug.");
}
CURRENT = new Build(shortHash, date);
}

4
core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
View File

@ -223,8 +223,8 @@ public class NodeStats extends BaseNodeResponse implements ToXContent {
http = HttpStats.readHttpStats(in);
}
breaker = AllCircuitBreakerStats.readOptionalAllCircuitBreakerStats(in);
scriptStats = in.readOptionalStreamable(new ScriptStats());
discoveryStats = in.readOptionalStreamable(new DiscoveryStats(null));
scriptStats = in.readOptionalStreamable(ScriptStats::new);
discoveryStats = in.readOptionalStreamable(() -> new DiscoveryStats(null));
}

8
core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
View File

@ -553,10 +553,10 @@ public class CommonStats implements Streamable, ToXContent {
if (in.readBoolean()) {
segments = SegmentsStats.readSegmentsStats(in);
}
translog = in.readOptionalStreamable(new TranslogStats());
suggest = in.readOptionalStreamable(new SuggestStats());
requestCache = in.readOptionalStreamable(new RequestCacheStats());
recoveryStats = in.readOptionalStreamable(new RecoveryStats());
translog = in.readOptionalStreamable(TranslogStats::new);
suggest = in.readOptionalStreamable(SuggestStats::new);
requestCache = in.readOptionalStreamable(RequestCacheStats::new);
recoveryStats = in.readOptionalStreamable(RecoveryStats::new);
}
@Override

2
core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
View File

@ -346,7 +346,7 @@ public class SearchRequest extends ActionRequest<SearchRequest> implements Indic
indicesOptions = IndicesOptions.readIndicesOptions(in);
requestCache = in.readOptionalBoolean();
template = in.readOptionalStreamable(new Template());
template = in.readOptionalStreamable(Template::new);
}
@Override

2
core/src/main/java/org/elasticsearch/cluster/routing/RestoreSource.java
View File

@ -68,7 +68,7 @@ public class RestoreSource implements Streamable, ToXContent {
}
public static RestoreSource readOptionalRestoreSource(StreamInput in) throws IOException {
return in.readOptionalStreamable(new RestoreSource());
return in.readOptionalStreamable(RestoreSource::new);
}
@Override

2
core/src/main/java/org/elasticsearch/cluster/routing/RoutingValidationException.java
View File

@ -39,7 +39,7 @@ public class RoutingValidationException extends RoutingException {
public RoutingValidationException(StreamInput in) throws IOException {
super(in);
validation = in.readOptionalStreamable(new RoutingTableValidation());
validation = in.readOptionalStreamable(RoutingTableValidation::new);
}
@Override

4
core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
View File

@ -53,6 +53,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import static org.elasticsearch.ElasticsearchException.readException;
import static org.elasticsearch.ElasticsearchException.readStackTrace;
@ -532,8 +533,9 @@ public abstract class StreamInput extends InputStream {
/**
* Serializes a potential null value.
*/
public <T extends Streamable> T readOptionalStreamable(T streamable) throws IOException {
public <T extends Streamable> T readOptionalStreamable(Supplier<T> supplier) throws IOException {
if (readBoolean()) {
T streamable = supplier.get();
streamable.readFrom(this);
return streamable;
} else {

2
core/src/main/java/org/elasticsearch/index/engine/CommitStats.java
View File

@ -62,7 +62,7 @@ public final class CommitStats implements Streamable, ToXContent {
}
public static CommitStats readOptionalCommitStatsFrom(StreamInput in) throws IOException {
return in.readOptionalStreamable(new CommitStats());
return in.readOptionalStreamable(CommitStats::new);
}

2
core/src/main/java/org/elasticsearch/indices/breaker/AllCircuitBreakerStats.java
View File

@ -57,7 +57,7 @@ public class AllCircuitBreakerStats implements Streamable, ToXContent {
}
public static AllCircuitBreakerStats readOptionalAllCircuitBreakerStats(StreamInput in) throws IOException {
AllCircuitBreakerStats stats = in.readOptionalStreamable(new AllCircuitBreakerStats());
AllCircuitBreakerStats stats = in.readOptionalStreamable(AllCircuitBreakerStats::new);
return stats;
}

2
core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerStats.java
View File

@ -74,7 +74,7 @@ public class CircuitBreakerStats implements Streamable, ToXContent {
}
public static CircuitBreakerStats readOptionalCircuitBreakerStats(StreamInput in) throws IOException {
CircuitBreakerStats stats = in.readOptionalStreamable(new CircuitBreakerStats());
CircuitBreakerStats stats = in.readOptionalStreamable(CircuitBreakerStats::new);
return stats;
}

20
core/src/main/java/org/elasticsearch/monitor/Probes.java Normal file
View File

@ -0,0 +1,20 @@
package org.elasticsearch.monitor;
import java.lang.management.OperatingSystemMXBean;
import java.lang.reflect.Method;
public class Probes {
public static short getLoadAndScaleToPercent(Method method, OperatingSystemMXBean osMxBean) {
if (method != null) {
try {
double load = (double) method.invoke(osMxBean);
if (load >= 0) {
return (short) (load * 100);
}
} catch (Throwable t) {
return -1;
}
}
return -1;
}
}

11
core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.monitor.os;
import org.apache.lucene.util.Constants;
import org.elasticsearch.monitor.Probes;
import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean;
@ -34,6 +35,7 @@ public class OsProbe {
private static final Method getFreeSwapSpaceSize;
private static final Method getTotalSwapSpaceSize;
private static final Method getSystemLoadAverage;
private static final Method getSystemCpuLoad;
static {
getFreePhysicalMemorySize = getMethod("getFreePhysicalMemorySize");
@ -41,6 +43,7 @@ public class OsProbe {
getFreeSwapSpaceSize = getMethod("getFreeSwapSpaceSize");
getTotalSwapSpaceSize = getMethod("getTotalSwapSpaceSize");
getSystemLoadAverage = getMethod("getSystemLoadAverage");
getSystemCpuLoad = getMethod("getSystemCpuLoad");
}
/**
@ -113,6 +116,10 @@ public class OsProbe {
}
}
public short getSystemCpuPercent() {
return Probes.getLoadAndScaleToPercent(getSystemCpuLoad, osMxBean);
}
private static class OsProbeHolder {
private final static OsProbe INSTANCE = new OsProbe();
}
@ -136,7 +143,9 @@ public class OsProbe {
public OsStats osStats() {
OsStats stats = new OsStats();
stats.timestamp = System.currentTimeMillis();
stats.loadAverage = getSystemLoadAverage();
stats.cpu = new OsStats.Cpu();
stats.cpu.percent = getSystemCpuPercent();
stats.cpu.loadAverage = getSystemLoadAverage();
OsStats.Mem mem = new OsStats.Mem();
mem.total = getTotalPhysicalMemorySize();

54
core/src/main/java/org/elasticsearch/monitor/os/OsStats.java
View File

@ -36,7 +36,7 @@ public class OsStats implements Streamable, ToXContent {
long timestamp;
double loadAverage = -1;
Cpu cpu = null;
Mem mem = null;
@ -49,10 +49,7 @@ public class OsStats implements Streamable, ToXContent {
return timestamp;
}
public double getLoadAverage() {
return loadAverage;
}
public Cpu getCpu() { return cpu; }
public Mem getMem() {
return mem;
@ -65,6 +62,8 @@ public class OsStats implements Streamable, ToXContent {
static final class Fields {
static final XContentBuilderString OS = new XContentBuilderString("os");
static final XContentBuilderString TIMESTAMP = new XContentBuilderString("timestamp");
static final XContentBuilderString CPU = new XContentBuilderString("cpu");
static final XContentBuilderString PERCENT = new XContentBuilderString("percent");
static final XContentBuilderString LOAD_AVERAGE = new XContentBuilderString("load_average");
static final XContentBuilderString MEM = new XContentBuilderString("mem");
@ -85,7 +84,12 @@ public class OsStats implements Streamable, ToXContent {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.OS);
builder.field(Fields.TIMESTAMP, getTimestamp());
builder.field(Fields.LOAD_AVERAGE, getLoadAverage());
if (cpu != null) {
builder.startObject(Fields.CPU);
builder.field(Fields.PERCENT, cpu.getPercent());
builder.field(Fields.LOAD_AVERAGE, cpu.getLoadAverage());
builder.endObject();
}
if (mem != null) {
builder.startObject(Fields.MEM);
@ -120,7 +124,7 @@ public class OsStats implements Streamable, ToXContent {
@Override
public void readFrom(StreamInput in) throws IOException {
timestamp = in.readVLong();
loadAverage = in.readDouble();
cpu = in.readOptionalStreamable(Cpu::new);
if (in.readBoolean()) {
mem = Mem.readMem(in);
}
@ -132,7 +136,7 @@ public class OsStats implements Streamable, ToXContent {
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(timestamp);
out.writeDouble(loadAverage);
out.writeOptionalStreamable(cpu);
if (mem == null) {
out.writeBoolean(false);
} else {
@ -147,6 +151,39 @@ public class OsStats implements Streamable, ToXContent {
}
}
public static class Cpu implements Streamable {
short percent = -1;
double loadAverage = -1;
Cpu() {}
public static Cpu readCpu(StreamInput in) throws IOException {
Cpu cpu = new Cpu();
cpu.readFrom(in);
return cpu;
}
@Override
public void readFrom(StreamInput in) throws IOException {
percent = in.readShort();
loadAverage = in.readDouble();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeShort(percent);
out.writeDouble(loadAverage);
}
public short getPercent() {
return percent;
}
public double getLoadAverage() {
return loadAverage;
}
}
public static class Swap implements Streamable {
long total = -1;
@ -230,5 +267,4 @@ public class OsStats implements Streamable, ToXContent {
private static short calculatePercentage(long used, long max) {
return max <= 0 ? 0 : (short) (Math.round((100d * used) / max));
}
}

13
core/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.monitor.process;
import org.elasticsearch.bootstrap.BootstrapInfo;
import org.elasticsearch.monitor.Probes;
import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean;
@ -88,17 +89,7 @@ public class ProcessProbe {
* Returns the process CPU usage in percent
*/
public short getProcessCpuPercent() {
if (getProcessCpuLoad != null) {
try {
double load = (double) getProcessCpuLoad.invoke(osMxBean);
if (load >= 0) {
return (short) (load * 100);
}
} catch (Throwable t) {
return -1;
}
}
return -1;
return Probes.getLoadAndScaleToPercent(getProcessCpuLoad, osMxBean);
}
/**

4
core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
View File

@ -130,6 +130,7 @@ public class RestNodesAction extends AbstractCatAction {
table.addCell("file_desc.percent", "default:false;alias:fdp,fileDescriptorPercent;text-align:right;desc:used file descriptor ratio");
table.addCell("file_desc.max", "default:false;alias:fdm,fileDescriptorMax;text-align:right;desc:max file descriptors");
table.addCell("cpu", "alias:cpu;text-align:right;desc:recent cpu usage");
table.addCell("load", "alias:l;text-align:right;desc:most recent load avg");
table.addCell("uptime", "default:false;alias:u;text-align:right;desc:node uptime");
table.addCell("node.role", "alias:r,role,dc,nodeRole;desc:d:data node, c:client node");
@ -258,7 +259,8 @@ public class RestNodesAction extends AbstractCatAction {
table.addCell(processStats == null ? null : calculatePercentage(processStats.getOpenFileDescriptors(), processStats.getMaxFileDescriptors()));
table.addCell(processStats == null ? null : processStats.getMaxFileDescriptors());
table.addCell(osStats == null ? null : String.format(Locale.ROOT, "%.2f", osStats.getLoadAverage()));
table.addCell(osStats == null ? null : Short.toString(osStats.getCpu().getPercent()));
table.addCell(osStats == null ? null : String.format(Locale.ROOT, "%.2f", osStats.getCpu().getLoadAverage()));
table.addCell(jvmStats == null ? null : jvmStats.getUptime());
table.addCell(node.clientNode() ? "c" : node.dataNode() ? "d" : "-");
table.addCell(masterId == null ? "x" : masterId.equals(node.id()) ? "*" : node.masterNode() ? "m" : "-");

2
core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
View File

@ -194,7 +194,7 @@ public class InternalAggregations implements Aggregations, ToXContent, Streamabl
}
public static InternalAggregations readOptionalAggregations(StreamInput in) throws IOException {
return in.readOptionalStreamable(new InternalAggregations());
return in.readOptionalStreamable(InternalAggregations::new);
}
@Override

4
core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
View File

@ -559,7 +559,7 @@ public class InternalSearchHit implements SearchHit {
score = in.readFloat();
id = in.readText();
type = in.readText();
nestedIdentity = in.readOptionalStreamable(new InternalNestedIdentity());
nestedIdentity = in.readOptionalStreamable(InternalNestedIdentity::new);
version = in.readLong();
source = in.readBytesReference();
if (source.length() == 0) {
@ -810,7 +810,7 @@ public class InternalSearchHit implements SearchHit {
public void readFrom(StreamInput in) throws IOException {
field = in.readOptionalText();
offset = in.readInt();
child = in.readOptionalStreamable(new InternalNestedIdentity());
child = in.readOptionalStreamable(InternalNestedIdentity::new);
}
@Override

2
core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
View File

@ -180,7 +180,7 @@ public class ShardSearchLocalRequest extends ContextAndHeaderHolder implements S
types = in.readStringArray();
filteringAliases = in.readStringArray();
nowInMillis = in.readVLong();
template = in.readOptionalStreamable(new Template());
template = in.readOptionalStreamable(Template::new);
requestCache = in.readOptionalBoolean();
}

2
core/src/main/java/org/elasticsearch/snapshots/RestoreInfo.java
View File

@ -190,7 +190,7 @@ public class RestoreInfo implements ToXContent, Streamable {
* @return restore info
*/
public static RestoreInfo readOptionalRestoreInfo(StreamInput in) throws IOException {
return in.readOptionalStreamable(new RestoreInfo());
return in.readOptionalStreamable(RestoreInfo::new);
}
}

2
core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
View File

@ -324,7 +324,7 @@ public class SnapshotInfo implements ToXContent, Streamable {
* @return deserialized snapshot info or null
*/
public static SnapshotInfo readOptionalSnapshotInfo(StreamInput in) throws IOException {
return in.readOptionalStreamable(new SnapshotInfo());
return in.readOptionalStreamable(SnapshotInfo::new);
}
}

12
core/src/test/java/org/elasticsearch/benchmark/monitor/os/OsProbeBenchmark.java
View File

@ -41,12 +41,10 @@ public class OsProbeBenchmark {
probe.getTotalSwapSpaceSize();
probe.getFreeSwapSpaceSize();
probe.getSystemLoadAverage();
probe.getSystemCpuPercent();
}
logger.info("--> warmed up");
logger.info("--> testing 'getTotalPhysicalMemorySize' method...");
long start = System.currentTimeMillis();
for (int i = 0; i < ITERATIONS; i++) {
@ -86,5 +84,13 @@ public class OsProbeBenchmark {
}
elapsed = System.currentTimeMillis() - start;
logger.info("--> total [{}] ms, avg [{}] ms", elapsed, (elapsed / (double)ITERATIONS));
logger.info("--> testing 'getSystemCpuPercent' method...");
start = System.currentTimeMillis();
for (int i = 0; i < ITERATIONS; i++) {
probe.getSystemCpuPercent();
}
elapsed = System.currentTimeMillis() - start;
logger.info("--> total [{}] ms, avg [{}] ms", elapsed, (elapsed / (double)ITERATIONS));
}
}

19
core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
View File

@ -1,3 +1,22 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.io.stream;
import org.elasticsearch.common.bytes.ByteBufferBytesReference;

13
core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java
View File

@ -22,13 +22,7 @@ package org.elasticsearch.monitor.os;
import org.apache.lucene.util.Constants;
import org.elasticsearch.test.ESTestCase;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.lessThan;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.hamcrest.Matchers.*;
public class OsProbeTests extends ESTestCase {
OsProbe probe = OsProbe.getInstance();
@ -47,12 +41,13 @@ public class OsProbeTests extends ESTestCase {
OsStats stats = probe.osStats();
assertNotNull(stats);
assertThat(stats.getTimestamp(), greaterThan(0L));
assertThat(stats.getCpu().getPercent(), anyOf(equalTo((short) -1), is(both(greaterThanOrEqualTo((short) 0)).and(lessThanOrEqualTo((short) 100)))));
if (Constants.WINDOWS) {
// Load average is always -1 on Windows platforms
assertThat(stats.getLoadAverage(), equalTo((double) -1));
assertThat(stats.getCpu().getLoadAverage(), equalTo((double) -1));
} else {
// Load average can be negative if not available or not computed yet, otherwise it should be >= 0
assertThat(stats.getLoadAverage(), anyOf(lessThan((double) 0), greaterThanOrEqualTo((double) 0)));
assertThat(stats.getCpu().getLoadAverage(), anyOf(lessThan((double) 0), greaterThanOrEqualTo((double) 0)));
}
assertNotNull(stats.getMem());

4
distribution/build.gradle
View File

@ -19,6 +19,7 @@
import org.apache.tools.ant.filters.FixCrLfFilter
import org.elasticsearch.gradle.precommit.DependencyLicensesTask
import org.elasticsearch.gradle.test.RunTask
import org.elasticsearch.gradle.MavenFilteringHack
// for deb/rpm
@ -196,4 +197,5 @@ DependencyLicensesTask.configure(project) {
mapping from: /jackson-.*/, to: 'jackson'
}
task run(type:org.elasticsearch.gradle.test.RunTask){}
RunTask.configure(project)

5
docs/reference/cluster/nodes-stats.asciidoc
View File

@ -128,7 +128,10 @@ the operating system:
`os.timestamp`::
Last time the operating system statistics have been refreshed
`os.load_average`::
`os.cpu.percent`::
Recent CPU usage for the whole system, or -1 if not supported
`os.cpu.load_average`::
System load average for the last minute, or -1 if not supported
`os.mem.total_in_bytes`::

12
docs/reference/migration/migrate_3_0.asciidoc
View File

@ -428,3 +428,15 @@ controls the backing queue for the thread pool and modifying this is an expert s
and high risk of being misused. The ability to change the thread pool type for any thread pool has been removed; do note
that it is still possible to adjust relevant thread pool parameters for each of the thread pools (e.g., depending on
the thread pool type, `keep_alive`, `queue_size`, etc.).
=== Adding system CPU percent to OS stats
The recent CPU usage (as a percent) has been added to the OS stats reported under the node stats API and the cat nodes
API. The breaking change here is that there is a new object in the "os" object in the node stats response. This object
is called "cpu" and includes "percent" and "load_average" as fields. This moves the "load_average" field that was
previously a top-level field in the "os" object to the "cpu" object. Additionally, the "cpu" field in the cat nodes API
response is output by default.
Finally, the API for org.elasticsearch.monitor.os.OsStats has changed. The `getLoadAverage` method has been removed. The
value for this can now be obtained from `OsStats.Cpu#getLoadAverage`. Additionally, the recent CPU usage can be obtained
from `OsStats.Cpu#getPercent`.

8
rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml
View File

@ -6,8 +6,8 @@
- match:
$body: |
/ #host ip heap.percent ram.percent load node.role master name
^ (\S+ \s+ (\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d*(\.\d+)? \s+ [-dc] \s+ [-*mx] \s+ (\S+\s?)+ \s+ \n)+ $/
/ #host ip heap.percent ram.percent cpu load node.role master name
^ (\S+ \s+ (\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ \d* \s+ (-)?\d*(\.\d+)? \s+ [-dc] \s+ [-*mx] \s+ (\S+\s?)+ \s+ \n)+ $/
- do:
cat.nodes:
@ -15,8 +15,8 @@
- match:
$body: |
/^ host \s+ ip \s+ heap\.percent \s+ ram\.percent \s+ load \s+ node\.role \s+ master \s+ name \s+ \n
(\S+ \s+ (\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d*(\.\d+)? \s+ [-dc] \s+ [-*mx] \s+ (\S+\s?)+ \s+ \n)+ $/
/^ host \s+ ip \s+ heap\.percent \s+ ram\.percent \s+ cpu \s+ load \s+ node\.role \s+ master \s+ name \s+ \n
(\S+ \s+ (\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ \d* \s+ (-)?\d*(\.\d+)? \s+ [-dc] \s+ [-*mx] \s+ (\S+\s?)+ \s+ \n)+ $/
- do:
cat.nodes: