honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ML] Remove C++/Java handshake code (elastic/x-pack-elasticsearch#3757) 

This is no longer needed following elastic/machine-learning-cpp#532

Original commit: elastic/x-pack-elasticsearch@76064b1539
This commit is contained in:
David Roberts 2018-01-26 18:04:11 +00:00 committed by GitHub
parent 621f4f20b0
commit 9351b984ac
5 changed files with 15 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessCtrl.java
View File

@ -6,7 +6,6 @@
package org.elasticsearch.xpack.ml.job.process;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.Randomness;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
@ -62,16 +61,10 @@ public class ProcessCtrl {
public static final Setting<Integer> MAX_ANOMALY_RECORDS_SETTING = Setting.intSetting("max.anomaly.records", DEFAULT_MAX_NUM_RECORDS,
Property.NodeScope);
/**
* This must match the value defined in CLicenseValidator::validate() in the C++ code
*/
static final long VALIDATION_NUMBER = 926213;
/*
* General arguments
*/
static final String JOB_ID_ARG = "--jobid=";
static final String LICENSE_VALIDATION_ARG = "--licenseValidation=";
/*
* Arguments used by both autodetect and normalize
@ -132,7 +125,6 @@ public class ProcessCtrl {
}
private ProcessCtrl() {
}
/**
@ -150,15 +142,13 @@ public class ProcessCtrl {
return rng.nextInt(SECONDS_IN_HOUR);
}
public static List<String> buildAutodetectCommand(Environment env, Settings settings, Job job, Logger logger, long controllerPid) {
public static List<String> buildAutodetectCommand(Environment env, Settings settings, Job job, Logger logger) {
List<String> command = new ArrayList<>();
command.add(AUTODETECT_PATH);
String jobId = JOB_ID_ARG + job.getId();
command.add(jobId);
command.add(makeLicenseArg(controllerPid));
AnalysisConfig analysisConfig = job.getAnalysisConfig();
if (analysisConfig != null) {
addIfNotNull(analysisConfig.getBucketSpan(), BUCKET_SPAN_ARG, command);
@ -232,7 +222,7 @@ public class ProcessCtrl {
private static void addIfNotNull(List<TimeValue> timeValues, String argKey, List<String> command) {
if (timeValues != null) {
addIfNotNull(timeValues.stream().map(v -> v.getSeconds()).collect(Collectors.toList()), argKey, command);
addIfNotNull(timeValues.stream().map(TimeValue::getSeconds).collect(Collectors.toList()), argKey, command);
}
}
@ -256,12 +246,11 @@ public class ProcessCtrl {
* Build the command to start the normalizer process.
*/
public static List<String> buildNormalizerCommand(Environment env, String jobId, String quantilesState, Integer bucketSpan,
boolean perPartitionNormalization, long controllerPid) throws IOException {
boolean perPartitionNormalization) throws IOException {
List<String> command = new ArrayList<>();
command.add(NORMALIZE_PATH);
addIfNotNull(bucketSpan, BUCKET_SPAN_ARG, command);
command.add(makeLicenseArg(controllerPid));
command.add(LENGTH_ENCODED_INPUT_ARG);
if (perPartitionNormalization) {
command.add(PER_PARTITION_NORMALIZATION);
@ -300,14 +289,4 @@ public class ProcessCtrl {
return stateFile;
}
/**
* The number must be equal to the daemon controller's PID modulo a magic number.
*/
private static String makeLicenseArg(long controllerPid) {
// Get a random int rather than long so we don't overflow when multiplying by VALIDATION_NUMBER
long rand = Randomness.get().nextInt();
long val = controllerPid + (((rand < 0) ? -rand : rand) + 1) * VALIDATION_NUMBER;
return LICENSE_VALIDATION_ARG + val;
}
}

5
plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java
View File

@ -31,7 +31,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeoutException;
/**
* The autodetect process builder.
@ -97,9 +96,9 @@ public class AutodetectBuilder {
/**
* Requests that the controller daemon start an autodetect process.
*/
public void build() throws IOException, TimeoutException {
public void build() throws IOException {
List<String> command = ProcessCtrl.buildAutodetectCommand(env, settings, job, logger, controller.getPid());
List<String> command = ProcessCtrl.buildAutodetectCommand(env, settings, job, logger);
buildLimits(command);
buildModelPlotConfig(command);

3
plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java
View File

@ -30,7 +30,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeoutException;
public class NativeAutodetectProcessFactory implements AutodetectProcessFactory {
@ -99,7 +98,7 @@ public class NativeAutodetectProcessFactory implements AutodetectProcessFactory
autodetectBuilder.build();
processPipes.connectStreams(PROCESS_STARTUP_TIMEOUT);
} catch (IOException | TimeoutException e) {
} catch (IOException e) {
String msg = "Failed to launch autodetect for job " + job.getId();
LOGGER.error(msg);
throw ExceptionsHelper.serverError(msg, e);

6
plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java
View File

@ -20,7 +20,6 @@ import java.time.Duration;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeoutException;
public class NativeNormalizerProcessFactory implements NormalizerProcessFactory {
@ -53,12 +52,11 @@ public class NativeNormalizerProcessFactory implements NormalizerProcessFactory
boolean perPartitionNormalization) {
try {
List<String> command = ProcessCtrl.buildNormalizerCommand(env, jobId, quantilesState, bucketSpan,
perPartitionNormalization, nativeController.getPid());
List<String> command = ProcessCtrl.buildNormalizerCommand(env, jobId, quantilesState, bucketSpan, perPartitionNormalization);
processPipes.addArgs(command);
nativeController.startProcess(command);
processPipes.connectStreams(PROCESS_STARTUP_TIMEOUT);
} catch (IOException | TimeoutException e) {
} catch (IOException e) {
String msg = "Failed to launch normalizer for job " + jobId;
LOGGER.error(msg);
throw ExceptionsHelper.serverError(msg, e);

37
plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/ProcessCtrlTests.java
View File

@ -26,9 +26,6 @@ import static org.elasticsearch.xpack.core.ml.job.config.JobTests.buildJobBuilde
public class ProcessCtrlTests extends ESTestCase {
private final Logger logger = Mockito.mock(Logger.class);
// 4194304 is the maximum possible PID on Linux according to
// http://web.archive.org/web/20111209081734/http://research.cs.wisc.edu/condor/condorg/linux_scalability.html
private final long pid = randomIntBetween(2, 4194304);
public void testBuildAutodetectCommand() {
Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
@ -52,8 +49,8 @@ public class ProcessCtrlTests extends ESTestCase {
dd.setTimeField("tf");
job.setDataDescription(dd);
List<String> command = ProcessCtrl.buildAutodetectCommand(env, settings, job.build(), logger, pid);
assertEquals(14, command.size());
List<String> command = ProcessCtrl.buildAutodetectCommand(env, settings, job.build(), logger);
assertEquals(13, command.size());
assertTrue(command.contains(ProcessCtrl.AUTODETECT_PATH));
assertTrue(command.contains(ProcessCtrl.BUCKET_SPAN_ARG + "120"));
assertTrue(command.contains(ProcessCtrl.LATENCY_ARG + "360"));
@ -65,7 +62,6 @@ public class ProcessCtrlTests extends ESTestCase {
assertTrue(command.contains(ProcessCtrl.maxAnomalyRecordsArg(settings)));
assertTrue(command.contains(ProcessCtrl.TIME_FIELD_ARG + "tf"));
assertTrue(hasValidLicense(command));
assertTrue(command.contains(ProcessCtrl.JOB_ID_ARG + "unit-test-job"));
assertTrue(command.contains(ProcessCtrl.PER_PARTITION_NORMALIZATION));
@ -81,7 +77,7 @@ public class ProcessCtrlTests extends ESTestCase {
Environment env = TestEnvironment.newEnvironment(settings);
Job.Builder job = buildJobBuilder("unit-test-job");
List<String> command = ProcessCtrl.buildAutodetectCommand(env, settings, job.build(), logger, pid);
List<String> command = ProcessCtrl.buildAutodetectCommand(env, settings, job.build(), logger);
assertTrue(command.contains(ProcessCtrl.TIME_FIELD_ARG + "time"));
}
@ -94,13 +90,13 @@ public class ProcessCtrlTests extends ESTestCase {
int expectedPersistInterval = 10800 + ProcessCtrl.calculateStaggeringInterval(job.getId());
List<String> command = ProcessCtrl.buildAutodetectCommand(env, settings, job.build(), logger, pid);
List<String> command = ProcessCtrl.buildAutodetectCommand(env, settings, job.build(), logger);
assertFalse(command.contains(ProcessCtrl.PERSIST_INTERVAL_ARG + expectedPersistInterval));
settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
env = TestEnvironment.newEnvironment(settings);
command = ProcessCtrl.buildAutodetectCommand(env, settings, job.build(), logger, pid);
command = ProcessCtrl.buildAutodetectCommand(env, settings, job.build(), logger);
assertTrue(command.contains(ProcessCtrl.PERSIST_INTERVAL_ARG + expectedPersistInterval));
}
@ -109,30 +105,11 @@ public class ProcessCtrlTests extends ESTestCase {
Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build());
String jobId = "unit-test-job";
List<String> command = ProcessCtrl.buildNormalizerCommand(env, jobId, null, 300, true, pid);
assertEquals(5, command.size());
List<String> command = ProcessCtrl.buildNormalizerCommand(env, jobId, null, 300, true);
assertEquals(4, command.size());
assertTrue(command.contains(ProcessCtrl.NORMALIZE_PATH));
assertTrue(command.contains(ProcessCtrl.BUCKET_SPAN_ARG + "300"));
assertTrue(hasValidLicense(command));
assertTrue(command.contains(ProcessCtrl.LENGTH_ENCODED_INPUT_ARG));
assertTrue(command.contains(ProcessCtrl.PER_PARTITION_NORMALIZATION));
}
private boolean hasValidLicense(List<String> command) throws NumberFormatException {
int matches = 0;
for (String arg : command) {
if (arg.startsWith(ProcessCtrl.LICENSE_VALIDATION_ARG)) {
++matches;
String[] argAndVal = arg.split("=");
if (argAndVal.length != 2) {
return false;
}
long val = Long.parseLong(argAndVal[1]);
if ((val % ProcessCtrl.VALIDATION_NUMBER) != (pid % ProcessCtrl.VALIDATION_NUMBER)) {
return false;
}
}
}
return matches == 1;
}
}