honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-24 13:55:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

[7.x][ML] Remove parsing of old progress format in DF Analytics (#55711) (#55720)

Browse Source 
Since #55580 we've introduced a new format for parsing progress
from the data frame analytics process. As the process is now
writing out progress in this new way, we can remove the parsing
of the old format.

Backport of #55711
This commit is contained in:
Dimitris Athanasiou 2020-04-24 16:50:56 +03:00 committed by GitHub
parent aa9a2bce37
commit 210b7f1b76
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 18 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
View File

@ -18,8 +18,8 @@ import org.elasticsearch.license.License;
import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification;
import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression;
import org.elasticsearch.xpack.core.ml.dataframe.stats.common.MemoryUsage;
import org.elasticsearch.xpack.core.ml.dataframe.stats.classification.ClassificationStats;
import org.elasticsearch.xpack.core.ml.dataframe.stats.common.MemoryUsage;
import org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection.OutlierDetectionStats;
import org.elasticsearch.xpack.core.ml.dataframe.stats.regression.RegressionStats;
import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
@ -171,13 +171,6 @@ public class AnalyticsResultProcessor {
phaseProgress.getProgressPercent());
statsHolder.getProgressTracker().analyzingPercent.set(phaseProgress.getProgressPercent());
}
// TODO remove after process is writing out phase_progress
Integer progressPercent = result.getProgressPercent();
if (progressPercent != null) {
statsHolder.getProgressTracker().analyzingPercent.set(progressPercent);
}
TrainedModelDefinition.Builder inferenceModelBuilder = result.getInferenceModelBuilder();
if (inferenceModelBuilder != null) {
createAndIndexInferenceModel(inferenceModelBuilder);

31
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/results/AnalyticsResult.java
View File

@ -30,7 +30,6 @@ public class AnalyticsResult implements ToXContentObject {
public static final ParseField TYPE = new ParseField("analytics_result");
private static final ParseField PHASE_PROGRESS = new ParseField("phase_progress");
private static final ParseField PROGRESS_PERCENT = new ParseField("progress_percent");
private static final ParseField INFERENCE_MODEL = new ParseField("inference_model");
private static final ParseField ANALYTICS_MEMORY_USAGE = new ParseField("analytics_memory_usage");
private static final ParseField OUTLIER_DETECTION_STATS = new ParseField("outlier_detection_stats");
@ -41,18 +40,16 @@ public class AnalyticsResult implements ToXContentObject {
a -> new AnalyticsResult(
(RowResults) a[0],
(PhaseProgress) a[1],
(Integer) a[2],
(TrainedModelDefinition.Builder) a[3],
(MemoryUsage) a[4],
(OutlierDetectionStats) a[5],
(ClassificationStats) a[6],
(RegressionStats) a[7]
(TrainedModelDefinition.Builder) a[2],
(MemoryUsage) a[3],
(OutlierDetectionStats) a[4],
(ClassificationStats) a[5],
(RegressionStats) a[6]
));
static {
PARSER.declareObject(optionalConstructorArg(), RowResults.PARSER, RowResults.TYPE);
PARSER.declareObject(optionalConstructorArg(), PhaseProgress.PARSER, PHASE_PROGRESS);
PARSER.declareInt(optionalConstructorArg(), PROGRESS_PERCENT);
// TODO change back to STRICT_PARSER once native side is aligned
PARSER.declareObject(optionalConstructorArg(), TrainedModelDefinition.LENIENT_PARSER, INFERENCE_MODEL);
PARSER.declareObject(optionalConstructorArg(), MemoryUsage.STRICT_PARSER, ANALYTICS_MEMORY_USAGE);
@ -63,10 +60,6 @@ public class AnalyticsResult implements ToXContentObject {
private final RowResults rowResults;
private final PhaseProgress phaseProgress;
// TODO remove after process is writing out phase_progress
private final Integer progressPercent;
private final TrainedModelDefinition.Builder inferenceModelBuilder;
private final TrainedModelDefinition inferenceModel;
private final MemoryUsage memoryUsage;
@ -76,7 +69,6 @@ public class AnalyticsResult implements ToXContentObject {
public AnalyticsResult(@Nullable RowResults rowResults,
@Nullable PhaseProgress phaseProgress,
@Nullable Integer progressPercent,
@Nullable TrainedModelDefinition.Builder inferenceModelBuilder,
@Nullable MemoryUsage memoryUsage,
@Nullable OutlierDetectionStats outlierDetectionStats,
@ -84,7 +76,6 @@ public class AnalyticsResult implements ToXContentObject {
@Nullable RegressionStats regressionStats) {
this.rowResults = rowResults;
this.phaseProgress = phaseProgress;
this.progressPercent = progressPercent;
this.inferenceModelBuilder = inferenceModelBuilder;
this.inferenceModel = inferenceModelBuilder == null ? null : inferenceModelBuilder.build();
this.memoryUsage = memoryUsage;
@ -101,10 +92,6 @@ public class AnalyticsResult implements ToXContentObject {
return phaseProgress;
}
public Integer getProgressPercent() {
return progressPercent;
}
public TrainedModelDefinition.Builder getInferenceModelBuilder() {
return inferenceModelBuilder;
}
@ -134,9 +121,6 @@ public class AnalyticsResult implements ToXContentObject {
if (phaseProgress != null) {
builder.field(PHASE_PROGRESS.getPreferredName(), phaseProgress);
}
if (progressPercent != null) {
builder.field(PROGRESS_PERCENT.getPreferredName(), progressPercent);
}
if (inferenceModel != null) {
builder.field(INFERENCE_MODEL.getPreferredName(),
inferenceModel,
@ -170,7 +154,6 @@ public class AnalyticsResult implements ToXContentObject {
AnalyticsResult that = (AnalyticsResult) other;
return Objects.equals(rowResults, that.rowResults)
&& Objects.equals(phaseProgress, that.phaseProgress)
&& Objects.equals(progressPercent, that.progressPercent)
&& Objects.equals(inferenceModel, that.inferenceModel)
&& Objects.equals(memoryUsage, that.memoryUsage)
&& Objects.equals(outlierDetectionStats, that.outlierDetectionStats)
@ -180,7 +163,7 @@ public class AnalyticsResult implements ToXContentObject {
@Override
public int hashCode() {
return Objects.hash(rowResults, phaseProgress, progressPercent, inferenceModel, memoryUsage, outlierDetectionStats,
classificationStats, regressionStats);
return Objects.hash(rowResults, phaseProgress, inferenceModel, memoryUsage, outlierDetectionStats, classificationStats,
regressionStats);
}
}

2
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java
View File

@ -58,7 +58,7 @@ public class AnalyticsProcessManagerTests extends ESTestCase {
private static final String CONFIG_ID = "config-id";
private static final int NUM_ROWS = 100;
private static final int NUM_COLS = 4;
private static final AnalyticsResult PROCESS_RESULT = new AnalyticsResult(null, null, null, null, null, null, null, null);
private static final AnalyticsResult PROCESS_RESULT = new AnalyticsResult(null, null, null, null, null, null, null);
private Client client;
private DataFrameAnalyticsAuditor auditor;

16
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
View File

@ -105,8 +105,8 @@ public class AnalyticsResultProcessorTests extends ESTestCase {
public void testProcess_GivenEmptyResults() {
givenDataFrameRows(2);
givenProcessResults(Arrays.asList(
new AnalyticsResult(null, null,50, null, null, null, null, null),
new AnalyticsResult(null, null, 100, null, null, null, null, null)));
new AnalyticsResult(null, null,null, null, null, null, null),
new AnalyticsResult(null, null, null, null, null, null, null)));
AnalyticsResultProcessor resultProcessor = createResultProcessor();
resultProcessor.process(process);
@ -121,8 +121,8 @@ public class AnalyticsResultProcessorTests extends ESTestCase {
givenDataFrameRows(2);
RowResults rowResults1 = mock(RowResults.class);
RowResults rowResults2 = mock(RowResults.class);
givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, null,50, null, null, null, null, null),
new AnalyticsResult(rowResults2, null, 100, null, null, null, null, null)));
givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, null,null, null, null, null, null),
new AnalyticsResult(rowResults2, null, null, null, null, null, null)));
AnalyticsResultProcessor resultProcessor = createResultProcessor();
resultProcessor.process(process);
@ -139,8 +139,8 @@ public class AnalyticsResultProcessorTests extends ESTestCase {
givenDataFrameRows(2);
RowResults rowResults1 = mock(RowResults.class);
RowResults rowResults2 = mock(RowResults.class);
givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, null,50, null, null, null, null, null),
new AnalyticsResult(rowResults2, null, 100, null, null, null, null, null)));
givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, null,null, null, null, null, null),
new AnalyticsResult(rowResults2, null, null, null, null, null, null)));
doThrow(new RuntimeException("some failure")).when(dataFrameRowsJoiner).processRowResults(any(RowResults.class));
@ -174,7 +174,7 @@ public class AnalyticsResultProcessorTests extends ESTestCase {
extractedFieldList.add(new DocValueField("baz", Collections.emptySet()));
TargetType targetType = analyticsConfig.getAnalysis() instanceof Regression ? TargetType.REGRESSION : TargetType.CLASSIFICATION;
TrainedModelDefinition.Builder inferenceModel = TrainedModelDefinitionTests.createRandomBuilder(targetType);
givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, null, inferenceModel, null, null, null, null)));
givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, inferenceModel, null, null, null, null)));
AnalyticsResultProcessor resultProcessor = createResultProcessor(extractedFieldList);
resultProcessor.process(process);
@ -238,7 +238,7 @@ public class AnalyticsResultProcessorTests extends ESTestCase {
TargetType targetType = analyticsConfig.getAnalysis() instanceof Regression ? TargetType.REGRESSION : TargetType.CLASSIFICATION;
TrainedModelDefinition.Builder inferenceModel = TrainedModelDefinitionTests.createRandomBuilder(targetType);
givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, null, inferenceModel, null, null, null, null)));
givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, inferenceModel, null, null, null, null)));
AnalyticsResultProcessor resultProcessor = createResultProcessor();
resultProcessor.process(process);

6
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/results/AnalyticsResultTests.java
View File

@ -43,7 +43,6 @@ public class AnalyticsResultTests extends AbstractXContentTestCase<AnalyticsResu
protected AnalyticsResult createTestInstance() {
RowResults rowResults = null;
PhaseProgress phaseProgress = null;
Integer progressPercent = null;
TrainedModelDefinition.Builder inferenceModel = null;
MemoryUsage memoryUsage = null;
OutlierDetectionStats outlierDetectionStats = null;
@ -55,9 +54,6 @@ public class AnalyticsResultTests extends AbstractXContentTestCase<AnalyticsResu
if (randomBoolean()) {
phaseProgress = new PhaseProgress(randomAlphaOfLength(10), randomIntBetween(0, 100));
}
if (randomBoolean()) {
progressPercent = randomIntBetween(0, 100);
}
if (randomBoolean()) {
inferenceModel = TrainedModelDefinitionTests.createRandomBuilder();
}
@ -73,7 +69,7 @@ public class AnalyticsResultTests extends AbstractXContentTestCase<AnalyticsResu
if (randomBoolean()) {
regressionStats = RegressionStatsTests.createRandom();
}
return new AnalyticsResult(rowResults, phaseProgress, progressPercent, inferenceModel, memoryUsage, outlierDetectionStats,
return new AnalyticsResult(rowResults, phaseProgress, inferenceModel, memoryUsage, outlierDetectionStats,
classificationStats, regressionStats);
}