From c34b63daddb1835b83100a183fef2e5b94951c8b Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 30 Jan 2017 17:47:23 -0500 Subject: [PATCH] Expand AbstractSerializingTestCase and AbstractWireSerializingTestCase to test diff serialization This commit adds two additional test cases that can be used to verify correct diff serialization in additional to binary and xcontent serialization. --- .../java/org/elasticsearch/cluster/Diff.java | 11 +- .../script/StoredScriptSource.java | 5 + .../script/ScriptMetaDataTests.java | 12 -- ...epositoriesMetaDataSerializationTests.java | 108 ++++++++++++++++ ...SnapshotsInProgressSerializationTests.java | 117 ++++++++++++++++++ ...AbstractDiffableSerializationTestCase.java | 49 ++++++++ ...ractDiffableWireSerializationTestCase.java | 47 +++++++ .../test/AbstractSerializingTestCase.java | 12 +- .../elasticsearch/test/DiffableTestUtils.java | 106 ++++++++++++++++ 9 files changed, 443 insertions(+), 24 deletions(-) create mode 100644 core/src/test/java/org/elasticsearch/snapshots/RepositoriesMetaDataSerializationTests.java create mode 100644 core/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java create mode 100644 test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableSerializationTestCase.java create mode 100644 test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableWireSerializationTestCase.java create mode 100644 test/framework/src/main/java/org/elasticsearch/test/DiffableTestUtils.java diff --git a/core/src/main/java/org/elasticsearch/cluster/Diff.java b/core/src/main/java/org/elasticsearch/cluster/Diff.java index 76535a4b763..4e980e36868 100644 --- a/core/src/main/java/org/elasticsearch/cluster/Diff.java +++ b/core/src/main/java/org/elasticsearch/cluster/Diff.java @@ -19,22 +19,15 @@ package org.elasticsearch.cluster; -import org.elasticsearch.common.io.stream.StreamOutput; - -import java.io.IOException; +import org.elasticsearch.common.io.stream.Writeable; /** * Represents difference between states of cluster state parts */ -public interface Diff { +public interface Diff extends Writeable { /** * Applies difference to the specified part and returns the resulted part */ T apply(T part); - - /** - * Writes the differences into the output stream - */ - void writeTo(StreamOutput out) throws IOException; } diff --git a/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java b/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java index acf7135424f..69fc1ed34cd 100644 --- a/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java +++ b/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java @@ -432,6 +432,11 @@ public class StoredScriptSource extends AbstractDiffable imp return builder; } + @Override + public boolean isFragment() { + return false; + } + /** * @return The language used for compiling this script. */ diff --git a/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java b/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java index 5365d75bdf9..e7365d667de 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java @@ -125,18 +125,6 @@ public class ScriptMetaDataTests extends AbstractSerializingTestCase { + + @Override + protected Custom createTestInstance() { + int numberOfRepositories = randomInt(10); + List entries = new ArrayList<>(); + for (int i = 0; i < numberOfRepositories; i++) { + entries.add(new RepositoryMetaData(randomAsciiOfLength(10), randomAsciiOfLength(10), randomSettings())); + } + entries.sort(Comparator.comparing(RepositoryMetaData::name)); + return new RepositoriesMetaData(entries.toArray(new RepositoryMetaData[entries.size()])); + } + + @Override + protected Writeable.Reader instanceReader() { + return RepositoriesMetaData::new; + } + + public Settings randomSettings() { + if (randomBoolean()) { + return Settings.EMPTY; + } else { + int numberOfSettings = randomInt(10); + Settings.Builder builder = Settings.builder(); + for (int i = 0; i < numberOfSettings; i++) { + builder.put(randomAsciiOfLength(10), randomAsciiOfLength(20)); + } + return builder.build(); + } + } + + @Override + protected Custom makeTestChanges(Custom testInstance) { + RepositoriesMetaData repositoriesMetaData = (RepositoriesMetaData) testInstance; + List repos = new ArrayList<>(repositoriesMetaData.repositories()); + if (randomBoolean() && repos.size() > 1) { + // remove some elements + int leaveElements = randomIntBetween(0, repositoriesMetaData.repositories().size() - 1); + repos = randomSubsetOf(leaveElements, repos.toArray(new RepositoryMetaData[leaveElements])); + } + if (randomBoolean()) { + // add some elements + int addElements = randomInt(10); + for (int i = 0; i < addElements; i++) { + repos.add(new RepositoryMetaData(randomAsciiOfLength(10), randomAsciiOfLength(10), randomSettings())); + } + } + return new RepositoriesMetaData(repos.toArray(new RepositoryMetaData[repos.size()])); + } + + @Override + protected Writeable.Reader> diffReader() { + return RepositoriesMetaData::readDiffFrom; + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); + } + + @Override + protected Custom doParseInstance(XContentParser parser) throws IOException { + assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); + RepositoriesMetaData repositoriesMetaData = RepositoriesMetaData.fromXContent(parser); + assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken()); + List repos = repositoriesMetaData.repositories(); + repos.sort(Comparator.comparing(RepositoryMetaData::name)); + return new RepositoriesMetaData(repos.toArray(new RepositoryMetaData[repos.size()])); + } + +} diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java new file mode 100644 index 00000000000..4b5a9677945 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java @@ -0,0 +1,117 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.snapshots; + +import org.elasticsearch.cluster.ClusterModule; +import org.elasticsearch.cluster.ClusterState.Custom; +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.cluster.SnapshotsInProgress.Entry; +import org.elasticsearch.cluster.SnapshotsInProgress.State; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.repositories.IndexId; +import org.elasticsearch.test.AbstractDiffableWireSerializationTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class SnapshotsInProgressSerializationTests extends AbstractDiffableWireSerializationTestCase { + + @Override + protected Custom createTestInstance() { + int numberOfSnapshots = randomInt(10); + List entries = new ArrayList<>(); + for (int i = 0; i < numberOfSnapshots; i++) { + entries.add(randomSnapshot()); + } + return new SnapshotsInProgress(entries); + } + + private Entry randomSnapshot() { + Snapshot snapshot = new Snapshot(randomAsciiOfLength(10), new SnapshotId(randomAsciiOfLength(10), randomAsciiOfLength(10))); + boolean includeGlobalState = randomBoolean(); + boolean partial = randomBoolean(); + State state = randomFrom(State.values()); + int numberOfIndices = randomIntBetween(0, 10); + List indices = new ArrayList<>(); + for (int i = 0; i < numberOfIndices; i++) { + indices.add(new IndexId(randomAsciiOfLength(10), randomAsciiOfLength(10))); + } + long startTime = randomLong(); + long repositoryStateId = randomLong(); + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + int shardsCount = randomIntBetween(0, 10); + for (int j = 0; j < shardsCount; j++) { + ShardId shardId = new ShardId(new Index(randomAsciiOfLength(10), randomAsciiOfLength(10)), randomIntBetween(0, 10)); + String nodeId = randomAsciiOfLength(10); + State shardState = randomFrom(State.values()); + builder.put(shardId, new SnapshotsInProgress.ShardSnapshotStatus(nodeId, shardState)); + } + ImmutableOpenMap shards = builder.build(); + return new Entry(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards); + } + + @Override + protected Writeable.Reader instanceReader() { + return SnapshotsInProgress::new; + } + + @Override + protected Custom makeTestChanges(Custom testInstance) { + SnapshotsInProgress snapshots = (SnapshotsInProgress) testInstance; + List entries = new ArrayList<>(snapshots.entries()); + if (randomBoolean() && entries.size() > 1) { + // remove some elements + int leaveElements = randomIntBetween(0, entries.size() - 1); + entries = randomSubsetOf(leaveElements, entries.toArray(new Entry[leaveElements])); + } + if (randomBoolean()) { + // add some elements + int addElements = randomInt(10); + for (int i = 0; i < addElements; i++) { + entries.add(randomSnapshot()); + } + } + if (randomBoolean()) { + // modify some elements + for (int i = 0; i < entries.size(); i++) { + if (randomBoolean()) { + entries.set(i, new Entry(entries.get(i), randomFrom(State.values()), entries.get(i).shards())); + } + } + } + return new SnapshotsInProgress(entries); + } + + @Override + protected Writeable.Reader> diffReader() { + return SnapshotsInProgress::readDiffFrom; + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); + } + +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableSerializationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableSerializationTestCase.java new file mode 100644 index 00000000000..feabca586b8 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableSerializationTestCase.java @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test; + +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.Diffable; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.ToXContent; + +import java.io.IOException; + +/** + * An abstract test case to ensure correct behavior of Diffable. + * + * This class can be used as a based class for tests of MetaData.Custom classes and other classes that support, + * Writable serialization, XContent-based serialization and is diffable. + */ +public abstract class AbstractDiffableSerializationTestCase & ToXContent> extends AbstractSerializingTestCase { + + /** + * Introduces random changes into the test object + */ + protected abstract T makeTestChanges(T testInstance); + + protected abstract Reader> diffReader(); + + public void testDiffableSerialization() throws IOException { + DiffableTestUtils.testDiffableSerialization(this::createTestInstance, this::makeTestChanges, getNamedWriteableRegistry(), + instanceReader(), diffReader()); + } + +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableWireSerializationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableWireSerializationTestCase.java new file mode 100644 index 00000000000..aa801308825 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableWireSerializationTestCase.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test; + +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.Diffable; +import org.elasticsearch.common.io.stream.Writeable.Reader; + +import java.io.IOException; + +/** + * An abstract test case to ensure correct behavior of Diffable. + * + * This class can be used as a based class for tests of ClusterState.Custom classes and other classes that support, + * Writable serialization and is diffable. + */ +public abstract class AbstractDiffableWireSerializationTestCase> extends AbstractWireSerializingTestCase { + /** + * Introduces random changes into the test object + */ + protected abstract T makeTestChanges(T testInstance); + + protected abstract Reader> diffReader(); + + public void testDiffableSerialization() throws IOException { + DiffableTestUtils.testDiffableSerialization(this::createTestInstance, this::makeTestChanges, getNamedWriteableRegistry(), + instanceReader(), diffReader()); + } + +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractSerializingTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractSerializingTestCase.java index 8d5556eeabb..392dcf1542a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractSerializingTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractSerializingTestCase.java @@ -70,11 +70,11 @@ public abstract class AbstractSerializingTestCase getAlternateVersions() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/DiffableTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/DiffableTestUtils.java new file mode 100644 index 00000000000..14d8ec02628 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/DiffableTestUtils.java @@ -0,0 +1,106 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test; + +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.Diffable; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.io.stream.Writeable.Reader; + +import java.io.IOException; +import java.util.function.Function; +import java.util.function.Supplier; + +import static org.elasticsearch.test.AbstractWireSerializingTestCase.NUMBER_OF_TEST_RUNS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; + +/** + * Utilities that simplify testing of diffable classes + */ +public final class DiffableTestUtils { + protected static final int NUMBER_OF_DIFF_TEST_RUNS = NUMBER_OF_TEST_RUNS; + + private DiffableTestUtils() { + + } + + /** + * Asserts that changes are applied correctly, i.e. that applying diffs to localInstance produces that object + * equal but not the same as the remoteChanges instance. + */ + public static > T assertDiffApplication(T remoteChanges, T localInstance, Diff diffs) { + T localChanges = diffs.apply(localInstance); + assertEquals(remoteChanges, localChanges); + assertEquals(remoteChanges.hashCode(), localChanges.hashCode()); + assertNotSame(remoteChanges, localChanges); + return localChanges; + } + + /** + * Simulates sending diffs over the wire + */ + public static T copyInstance(T diffs, NamedWriteableRegistry namedWriteableRegistry, + Reader reader) throws IOException { + try (BytesStreamOutput output = new BytesStreamOutput()) { + diffs.writeTo(output); + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { + return reader.read(in); + } + } + } + + /** + * Tests making random changes to an object, calculating diffs for these changes, sending this + * diffs over the wire and appling these diffs on the other side. + */ + public static > void testDiffableSerialization(Supplier testInstance, + Function modifier, + NamedWriteableRegistry namedWriteableRegistry, + Reader reader, + Reader> diffReader) throws IOException { + T remoteInstance = testInstance.get(); + T localInstance = assertSerialization(remoteInstance, namedWriteableRegistry, reader); + for (int runs = 0; runs < NUMBER_OF_DIFF_TEST_RUNS; runs++) { + T remoteChanges = modifier.apply(remoteInstance); + Diff remoteDiffs = remoteChanges.diff(remoteInstance); + Diff localDiffs = copyInstance(remoteDiffs, namedWriteableRegistry, diffReader); + localInstance = assertDiffApplication(remoteChanges, localInstance, localDiffs); + remoteInstance = remoteChanges; + } + } + + /** + * Asserts that testInstance can be correctly. + */ + public static T assertSerialization(T testInstance, NamedWriteableRegistry namedWriteableRegistry, + Reader reader) throws IOException { + T deserializedInstance = copyInstance(testInstance, namedWriteableRegistry, reader); + assertEquals(testInstance, deserializedInstance); + assertEquals(testInstance.hashCode(), deserializedInstance.hashCode()); + assertNotSame(testInstance, deserializedInstance); + return deserializedInstance; + } + +}