Expand AbstractSerializingTestCase and AbstractWireSerializingTestCase to test diff serialization

This commit adds two additional test cases that can be used to verify correct diff serialization in additional to binary and xcontent serialization.

core/src/main/java/org/elasticsearch/cluster/Diff.java

@@ -19,22 +19,15 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
-
-import java.io.IOException;
 
 /**
  * Represents difference between states of cluster state parts
  */
-public interface Diff<T> {
+public interface Diff<T> extends Writeable {
 
     /**
      * Applies difference to the specified part and returns the resulted part
      */
     T apply(T part);
-
-    /**
-     * Writes the differences into the output stream
-     */
-    void writeTo(StreamOutput out) throws IOException;
 }

core/src/main/java/org/elasticsearch/script/StoredScriptSource.java

@@ -432,6 +432,11 @@ public class StoredScriptSource extends AbstractDiffable<StoredScriptSource> imp
         return builder;
     }
 
+    @Override
+    public boolean isFragment() {
+        return false;
+    }
+
     /**
      * @return The language used for compiling this script.
      */

core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java

@@ -125,18 +125,6 @@ public class ScriptMetaDataTests extends AbstractSerializingTestCase<ScriptMetaD
         return ScriptMetaData::new;
     }
 
-    @Override
-    protected XContentBuilder toXContent(ScriptMetaData instance, XContentType contentType) throws IOException {
-        XContentBuilder builder = XContentFactory.contentBuilder(contentType);
-        if (randomBoolean()) {
-            builder.prettyPrint();
-        }
-        builder.startObject();
-        instance.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        builder.endObject();
-        return builder;
-    }
-
     @Override
     protected ScriptMetaData doParseInstance(XContentParser parser) {
         try {

core/src/test/java/org/elasticsearch/snapshots/RepositoriesMetaDataSerializationTests.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.metadata.MetaData.Custom;
+import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractDiffableSerializationTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+public class RepositoriesMetaDataSerializationTests extends AbstractDiffableSerializationTestCase<Custom> {
+
+    @Override
+    protected Custom createTestInstance() {
+        int numberOfRepositories = randomInt(10);
+        List<RepositoryMetaData> entries = new ArrayList<>();
+        for (int i = 0; i < numberOfRepositories; i++) {
+            entries.add(new RepositoryMetaData(randomAsciiOfLength(10), randomAsciiOfLength(10), randomSettings()));
+        }
+        entries.sort(Comparator.comparing(RepositoryMetaData::name));
+        return new RepositoriesMetaData(entries.toArray(new RepositoryMetaData[entries.size()]));
+    }
+
+    @Override
+    protected Writeable.Reader<Custom> instanceReader() {
+        return RepositoriesMetaData::new;
+    }
+
+    public Settings randomSettings() {
+        if (randomBoolean()) {
+            return Settings.EMPTY;
+        } else {
+            int numberOfSettings = randomInt(10);
+            Settings.Builder builder = Settings.builder();
+            for (int i = 0; i < numberOfSettings; i++) {
+                builder.put(randomAsciiOfLength(10), randomAsciiOfLength(20));
+            }
+            return builder.build();
+        }
+    }
+
+    @Override
+    protected Custom makeTestChanges(Custom testInstance) {
+        RepositoriesMetaData repositoriesMetaData = (RepositoriesMetaData) testInstance;
+        List<RepositoryMetaData> repos = new ArrayList<>(repositoriesMetaData.repositories());
+        if (randomBoolean() && repos.size() > 1) {
+            // remove some elements
+            int leaveElements = randomIntBetween(0, repositoriesMetaData.repositories().size() - 1);
+            repos = randomSubsetOf(leaveElements, repos.toArray(new RepositoryMetaData[leaveElements]));
+        }
+        if (randomBoolean()) {
+            // add some elements
+            int addElements = randomInt(10);
+            for (int i = 0; i < addElements; i++) {
+                repos.add(new RepositoryMetaData(randomAsciiOfLength(10), randomAsciiOfLength(10), randomSettings()));
+            }
+        }
+        return new RepositoriesMetaData(repos.toArray(new RepositoryMetaData[repos.size()]));
+    }
+
+    @Override
+    protected Writeable.Reader<Diff<Custom>> diffReader() {
+        return RepositoriesMetaData::readDiffFrom;
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
+    }
+
+    @Override
+    protected Custom doParseInstance(XContentParser parser) throws IOException {
+        assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+        RepositoriesMetaData repositoriesMetaData = RepositoriesMetaData.fromXContent(parser);
+        assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+        List<RepositoryMetaData> repos = repositoriesMetaData.repositories();
+        repos.sort(Comparator.comparing(RepositoryMetaData::name));
+        return new RepositoriesMetaData(repos.toArray(new RepositoryMetaData[repos.size()]));
+    }
+
+}

core/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java

@@ -0,0 +1,117 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.ClusterState.Custom;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.SnapshotsInProgress;
+import org.elasticsearch.cluster.SnapshotsInProgress.Entry;
+import org.elasticsearch.cluster.SnapshotsInProgress.State;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.repositories.IndexId;
+import org.elasticsearch.test.AbstractDiffableWireSerializationTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class SnapshotsInProgressSerializationTests extends AbstractDiffableWireSerializationTestCase<Custom> {
+
+    @Override
+    protected Custom createTestInstance() {
+        int numberOfSnapshots = randomInt(10);
+        List<Entry> entries = new ArrayList<>();
+        for (int i = 0; i < numberOfSnapshots; i++) {
+            entries.add(randomSnapshot());
+        }
+        return new SnapshotsInProgress(entries);
+    }
+
+    private Entry randomSnapshot() {
+        Snapshot snapshot = new Snapshot(randomAsciiOfLength(10), new SnapshotId(randomAsciiOfLength(10), randomAsciiOfLength(10)));
+        boolean includeGlobalState = randomBoolean();
+        boolean partial = randomBoolean();
+        State state = randomFrom(State.values());
+        int numberOfIndices = randomIntBetween(0, 10);
+        List<IndexId> indices = new ArrayList<>();
+        for (int i = 0; i < numberOfIndices; i++) {
+            indices.add(new IndexId(randomAsciiOfLength(10), randomAsciiOfLength(10)));
+        }
+        long startTime = randomLong();
+        long repositoryStateId = randomLong();
+        ImmutableOpenMap.Builder<ShardId, SnapshotsInProgress.ShardSnapshotStatus> builder = ImmutableOpenMap.builder();
+        int shardsCount = randomIntBetween(0, 10);
+        for (int j = 0; j < shardsCount; j++) {
+            ShardId shardId = new ShardId(new Index(randomAsciiOfLength(10), randomAsciiOfLength(10)), randomIntBetween(0, 10));
+            String nodeId = randomAsciiOfLength(10);
+            State shardState = randomFrom(State.values());
+            builder.put(shardId, new SnapshotsInProgress.ShardSnapshotStatus(nodeId, shardState));
+        }
+        ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards = builder.build();
+        return new Entry(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards);
+    }
+
+    @Override
+    protected Writeable.Reader<Custom> instanceReader() {
+        return SnapshotsInProgress::new;
+    }
+
+    @Override
+    protected Custom makeTestChanges(Custom testInstance) {
+        SnapshotsInProgress snapshots = (SnapshotsInProgress) testInstance;
+        List<Entry> entries = new ArrayList<>(snapshots.entries());
+        if (randomBoolean() && entries.size() > 1) {
+            // remove some elements
+            int leaveElements = randomIntBetween(0, entries.size() - 1);
+            entries = randomSubsetOf(leaveElements, entries.toArray(new Entry[leaveElements]));
+        }
+        if (randomBoolean()) {
+            // add some elements
+            int addElements = randomInt(10);
+            for (int i = 0; i < addElements; i++) {
+                entries.add(randomSnapshot());
+            }
+        }
+        if (randomBoolean()) {
+            // modify some elements
+            for (int i = 0; i < entries.size(); i++) {
+                if (randomBoolean()) {
+                    entries.set(i, new Entry(entries.get(i), randomFrom(State.values()), entries.get(i).shards()));
+                }
+            }
+        }
+        return new SnapshotsInProgress(entries);
+    }
+
+    @Override
+    protected Writeable.Reader<Diff<Custom>> diffReader() {
+        return SnapshotsInProgress::readDiffFrom;
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
+    }
+
+}

test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableSerializationTestCase.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.Diffable;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.xcontent.ToXContent;
+
+import java.io.IOException;
+
+/**
+ * An abstract test case to ensure correct behavior of Diffable.
+ *
+ * This class can be used as a based class for tests of MetaData.Custom classes and other classes that support,
+ * Writable serialization, XContent-based serialization and is diffable.
+ */
+public abstract class AbstractDiffableSerializationTestCase<T extends Diffable<T> & ToXContent> extends AbstractSerializingTestCase<T> {
+
+    /**
+     *  Introduces random changes into the test object
+     */
+    protected abstract T makeTestChanges(T testInstance);
+
+    protected abstract Reader<Diff<T>> diffReader();
+
+    public void testDiffableSerialization() throws IOException {
+        DiffableTestUtils.testDiffableSerialization(this::createTestInstance, this::makeTestChanges, getNamedWriteableRegistry(),
+            instanceReader(), diffReader());
+    }
+
+}

test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableWireSerializationTestCase.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.Diffable;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+
+import java.io.IOException;
+
+/**
+ * An abstract test case to ensure correct behavior of Diffable.
+ *
+ * This class can be used as a based class for tests of ClusterState.Custom classes and other classes that support,
+ * Writable serialization and is diffable.
+ */
+public abstract class AbstractDiffableWireSerializationTestCase<T extends Diffable<T>> extends AbstractWireSerializingTestCase<T> {
+    /**
+     *  Introduces random changes into the test object
+     */
+    protected abstract T makeTestChanges(T testInstance);
+
+    protected abstract Reader<Diff<T>> diffReader();
+
+    public void testDiffableSerialization() throws IOException {
+        DiffableTestUtils.testDiffableSerialization(this::createTestInstance, this::makeTestChanges, getNamedWriteableRegistry(),
+            instanceReader(), diffReader());
+    }
+
+}

test/framework/src/main/java/org/elasticsearch/test/AbstractSerializingTestCase.java

@@ -70,7 +70,7 @@ public abstract class AbstractSerializingTestCase<T extends ToXContent & Writeab
     /**
      * Parses to a new instance using the provided {@link XContentParser}
      */
-    protected abstract T doParseInstance(XContentParser parser);
+    protected abstract T doParseInstance(XContentParser parser) throws IOException;
 
     /**
      * Renders the provided instance in XContent
@@ -86,7 +86,13 @@ public abstract class AbstractSerializingTestCase<T extends ToXContent & Writeab
         if (randomBoolean()) {
             builder.prettyPrint();
         }
+        if (instance.isFragment()) {
+            builder.startObject();
+        }
         instance.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        if (instance.isFragment()) {
+            builder.endObject();
+        }
         return builder;
     }
 

test/framework/src/main/java/org/elasticsearch/test/DiffableTestUtils.java

@@ -0,0 +1,106 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.Diffable;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+
+import java.io.IOException;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.test.AbstractWireSerializingTestCase.NUMBER_OF_TEST_RUNS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+
+/**
+ * Utilities that simplify testing of diffable classes
+ */
+public final class DiffableTestUtils {
+    protected static final int NUMBER_OF_DIFF_TEST_RUNS = NUMBER_OF_TEST_RUNS;
+
+    private DiffableTestUtils() {
+
+    }
+
+    /**
+     * Asserts that changes are applied correctly, i.e. that applying diffs to localInstance produces that object
+     * equal but not the same as the remoteChanges instance.
+     */
+    public static <T extends Diffable<T>> T assertDiffApplication(T remoteChanges, T localInstance, Diff<T> diffs) {
+        T localChanges = diffs.apply(localInstance);
+        assertEquals(remoteChanges, localChanges);
+        assertEquals(remoteChanges.hashCode(), localChanges.hashCode());
+        assertNotSame(remoteChanges, localChanges);
+        return localChanges;
+    }
+
+    /**
+     * Simulates sending diffs over the wire
+     */
+    public static <T extends Writeable> T copyInstance(T diffs, NamedWriteableRegistry namedWriteableRegistry,
+                                                                  Reader<T> reader) throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            diffs.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
+                return reader.read(in);
+            }
+        }
+    }
+
+    /**
+     * Tests making random changes to an object, calculating diffs for these changes, sending this
+     * diffs over the wire and appling these diffs on the other side.
+     */
+    public static <T extends Diffable<T>> void testDiffableSerialization(Supplier<T> testInstance,
+                                                                         Function<T, T> modifier,
+                                                                         NamedWriteableRegistry namedWriteableRegistry,
+                                                                         Reader<T> reader,
+                                                                         Reader<Diff<T>> diffReader) throws IOException {
+        T remoteInstance = testInstance.get();
+        T localInstance = assertSerialization(remoteInstance, namedWriteableRegistry, reader);
+        for (int runs = 0; runs < NUMBER_OF_DIFF_TEST_RUNS; runs++) {
+            T remoteChanges = modifier.apply(remoteInstance);
+            Diff<T> remoteDiffs = remoteChanges.diff(remoteInstance);
+            Diff<T> localDiffs = copyInstance(remoteDiffs, namedWriteableRegistry, diffReader);
+            localInstance = assertDiffApplication(remoteChanges, localInstance, localDiffs);
+            remoteInstance = remoteChanges;
+        }
+    }
+
+    /**
+     * Asserts that testInstance can be correctly.
+     */
+    public static  <T extends Writeable> T assertSerialization(T testInstance, NamedWriteableRegistry namedWriteableRegistry,
+                                                          Reader<T> reader) throws IOException {
+        T deserializedInstance = copyInstance(testInstance, namedWriteableRegistry, reader);
+        assertEquals(testInstance, deserializedInstance);
+        assertEquals(testInstance.hashCode(), deserializedInstance.hashCode());
+        assertNotSame(testInstance, deserializedInstance);
+        return deserializedInstance;
+    }
+
+}