honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into ccr 

* master:
  Removing erroneous repeat
  Adapt bwc versions after backporting #30983 to 6.4
  [Tests] Muting RatedRequestsTests#testXContentParsingIsNotLenient
  TEST:  Retry synced-flush if ongoing ops on primary (#30978)
  Fix docs build.
  Only auto-update license signature if all nodes ready (#30859)
  Add BlobContainer.writeBlobAtomic() (#30902)
  Add a doc value format to binary fields. (#30860)
This commit is contained in:
Nhat Nguyen 2018-06-05 10:58:35 -04:00
parent 530089f132 14c40885be
commit 2b5c0d77e9
40 changed files with 412 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
buildSrc/src/main/resources/checkstyle_suppressions.xml
View File

@ -505,8 +505,6 @@
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]settings[/\\]ClusterSettingsIT.java" checks="LineLength" />
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]shards[/\\]ClusterSearchShardsIT.java" checks="LineLength" />
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]structure[/\\]RoutingIteratorTests.java" checks="LineLength" />
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]FsBlobStoreContainerTests.java" checks="LineLength" />
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]FsBlobStoreTests.java" checks="LineLength" />
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]breaker[/\\]MemoryCircuitBreakerTests.java" checks="LineLength" />
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]geo[/\\]ShapeBuilderTests.java" checks="LineLength" />
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]hash[/\\]MessageDigestsTests.java" checks="LineLength" />

1
docs/reference/query-dsl/query-string-syntax.asciidoc
View File

@ -254,7 +254,6 @@ would look like this:
}
}
****
===== Grouping

1
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java
View File

@ -131,6 +131,7 @@ public class RatedRequestsTests extends ESTestCase {
}
}
@AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/31104")
public void testXContentParsingIsNotLenient() throws IOException {
RatedRequest testItem = createTestItem(randomBoolean());
XContentType xContentType = randomFrom(XContentType.values());

4
server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
View File

@ -63,7 +63,7 @@ public class ClusterRerouteResponse extends AcknowledgedResponse implements ToXC
@Override
public void readFrom(StreamInput in) throws IOException {
if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
super.readFrom(in);
state = ClusterState.readFrom(in, null);
explanations = RoutingExplanations.readFrom(in);
@ -76,7 +76,7 @@ public class ClusterRerouteResponse extends AcknowledgedResponse implements ToXC
@Override
public void writeTo(StreamOutput out) throws IOException {
if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
super.writeTo(out);
state.writeTo(out);
RoutingExplanations.writeTo(explanations, out);

4
server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java
View File

@ -68,7 +68,7 @@ public class ClusterUpdateSettingsResponse extends AcknowledgedResponse {
@Override
public void readFrom(StreamInput in) throws IOException {
if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
super.readFrom(in);
transientSettings = Settings.readSettingsFromStream(in);
persistentSettings = Settings.readSettingsFromStream(in);
@ -89,7 +89,7 @@ public class ClusterUpdateSettingsResponse extends AcknowledgedResponse {
@Override
public void writeTo(StreamOutput out) throws IOException {
if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
super.writeTo(out);
Settings.writeSettingsToStream(transientSettings, out);
Settings.writeSettingsToStream(persistentSettings, out);

4
server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
View File

@ -115,7 +115,7 @@ public final class RolloverResponse extends ShardsAcknowledgedResponse implement
@Override
public void readFrom(StreamInput in) throws IOException {
if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
super.readFrom(in);
oldIndex = in.readString();
newIndex = in.readString();
@ -144,7 +144,7 @@ public final class RolloverResponse extends ShardsAcknowledgedResponse implement
@Override
public void writeTo(StreamOutput out) throws IOException {
if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
super.writeTo(out);
out.writeString(oldIndex);
out.writeString(newIndex);

23
server/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java
View File

@ -74,6 +74,29 @@ public interface BlobContainer {
*/
void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException;
/**
* Reads blob content from the input stream and writes it to the container in a new blob with the given name,
* using an atomic write operation if the implementation supports it. When the BlobContainer implementation
* does not provide a specific implementation of writeBlobAtomic(String, InputStream, long), then
* the {@link #writeBlob(String, InputStream, long)} method is used.
*
* This method assumes the container does not already contain a blob of the same blobName. If a blob by the
* same name already exists, the operation will fail and an {@link IOException} will be thrown.
*
* @param blobName
* The name of the blob to write the contents of the input stream to.
* @param inputStream
* The input stream from which to retrieve the bytes to write to the blob.
* @param blobSize
* The size of the blob to be written, in bytes. It is implementation dependent whether
* this value is used in writing the blob to the repository.
* @throws FileAlreadyExistsException if a blob by the same name already exists
* @throws IOException if the input stream could not be read, or the target blob could not be written to.
*/
default void writeBlobAtomic(final String blobName, final InputStream inputStream, final long blobSize) throws IOException {
writeBlob(blobName, inputStream, blobSize);
}
/**
* Deletes a blob with giving name, if the blob exists. If the blob does not exist,
* this method throws a NoSuchFileException.

48
server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
View File

@ -19,11 +19,12 @@
package org.elasticsearch.common.blobstore.fs;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.blobstore.BlobMetaData;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.core.internal.io.Streams;
import java.io.BufferedInputStream;
@ -56,8 +57,9 @@ import static java.util.Collections.unmodifiableMap;
*/
public class FsBlobContainer extends AbstractBlobContainer {
protected final FsBlobStore blobStore;
private static final String TEMP_FILE_PREFIX = "pending-";
protected final FsBlobStore blobStore;
protected final Path path;
public FsBlobContainer(FsBlobStore blobStore, BlobPath blobPath, Path path) {
@ -131,6 +133,48 @@ public class FsBlobContainer extends AbstractBlobContainer {
IOUtils.fsync(path, true);
}
@Override
public void writeBlobAtomic(final String blobName, final InputStream inputStream, final long blobSize) throws IOException {
final String tempBlob = tempBlobName(blobName);
final Path tempBlobPath = path.resolve(tempBlob);
try {
try (OutputStream outputStream = Files.newOutputStream(tempBlobPath, StandardOpenOption.CREATE_NEW)) {
Streams.copy(inputStream, outputStream);
}
IOUtils.fsync(tempBlobPath, false);
final Path blobPath = path.resolve(blobName);
// If the target file exists then Files.move() behaviour is implementation specific
// the existing file might be replaced or this method fails by throwing an IOException.
if (Files.exists(blobPath)) {
throw new FileAlreadyExistsException("blob [" + blobPath + "] already exists, cannot overwrite");
}
Files.move(tempBlobPath, blobPath, StandardCopyOption.ATOMIC_MOVE);
} catch (IOException ex) {
try {
deleteBlobIgnoringIfNotExists(tempBlob);
} catch (IOException e) {
ex.addSuppressed(e);
}
throw ex;
} finally {
IOUtils.fsync(path, true);
}
}
public static String tempBlobName(final String blobName) {
return "pending-" + blobName + "-" + UUIDs.randomBase64UUID();
}
/**
* Returns true if the blob is a leftover temporary blob.
*
* The temporary blobs might be left after failed atomic write operation.
*/
public static boolean isTempBlobName(final String blobName) {
return blobName.startsWith(TEMP_FILE_PREFIX);
}
@Override
public void move(String source, String target) throws IOException {
Path sourcePath = path.resolve(source);

6
server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
View File

@ -40,6 +40,8 @@ import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.QueryShardException;
import org.elasticsearch.search.DocValueFormat;
import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.util.Base64;
@ -104,6 +106,10 @@ public class BinaryFieldMapper extends FieldMapper {
return CONTENT_TYPE;
}
@Override
public DocValueFormat docValueFormat(String format, DateTimeZone timeZone) {
return DocValueFormat.BINARY;
}
@Override
public BytesReference valueForDisplay(Object value) {

11
server/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
View File

@ -507,18 +507,7 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
if (indexShard.routingEntry().primary() == false) {
throw new IllegalStateException("[" + request.shardId() +"] expected a primary shard");
}
if (Assertions.ENABLED) {
if (logger.isTraceEnabled()) {
logger.trace("in flight operations {}, acquirers {}", indexShard.getActiveOperationsCount(), indexShard.getActiveOperations());
}
}
int opCount = indexShard.getActiveOperationsCount();
// Need to snapshot the debug info twice as it's updated concurrently with the permit count.
if (Assertions.ENABLED) {
if (logger.isTraceEnabled()) {
logger.trace("in flight operations {}, acquirers {}", indexShard.getActiveOperationsCount(), indexShard.getActiveOperations());
}
}
return new InFlightOpsResponse(opCount);
}

19
server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
View File

@ -50,6 +50,7 @@ import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.blobstore.BlobMetaData;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.BlobStore;
import org.elasticsearch.common.blobstore.fs.FsBlobContainer;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.Tuple;
@ -555,10 +556,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
String blobName = "master.dat";
BytesArray bytes = new BytesArray(testBytes);
try (InputStream stream = bytes.streamInput()) {
testContainer.writeBlob(blobName + "-temp", stream, bytes.length());
testContainer.writeBlobAtomic(blobName, stream, bytes.length());
}
// Make sure that move is supported
testContainer.move(blobName + "-temp", blobName);
return seed;
}
} catch (IOException exp) {
@ -774,18 +773,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
}
private void writeAtomic(final String blobName, final BytesReference bytesRef) throws IOException {
final String tempBlobName = "pending-" + blobName + "-" + UUIDs.randomBase64UUID();
try (InputStream stream = bytesRef.streamInput()) {
snapshotsBlobContainer.writeBlob(tempBlobName, stream, bytesRef.length());
snapshotsBlobContainer.move(tempBlobName, blobName);
} catch (IOException ex) {
// temporary blob creation or move failed - try cleaning up
try {
snapshotsBlobContainer.deleteBlobIgnoringIfNotExists(tempBlobName);
} catch (IOException e) {
ex.addSuppressed(e);
}
throw ex;
snapshotsBlobContainer.writeBlobAtomic(blobName, stream, bytesRef.length());
}
}
@ -955,7 +944,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
// Delete temporary index files first, as we might otherwise fail in the next step creating the new index file if an earlier
// attempt to write an index file with this generation failed mid-way after creating the temporary file.
for (final String blobName : blobs.keySet()) {
if (indexShardSnapshotsFormat.isTempBlobName(blobName)) {
if (FsBlobContainer.isTempBlobName(blobName)) {
try {
blobContainer.deleteBlobIgnoringIfNotExists(blobName);
} catch (IOException e) {

71
server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
View File

@ -23,6 +23,7 @@ import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexFormatTooNewException;
import org.apache.lucene.index.IndexFormatTooOldException;
import org.apache.lucene.store.OutputStreamIndexOutput;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.bytes.BytesArray;
@ -52,8 +53,6 @@ import java.util.Locale;
*/
public class ChecksumBlobStoreFormat<T extends ToXContent> extends BlobStoreFormat<T> {
private static final String TEMP_FILE_PREFIX = "pending-";
private static final XContentType DEFAULT_X_CONTENT_TYPE = XContentType.SMILE;
// The format version
@ -120,7 +119,7 @@ public class ChecksumBlobStoreFormat<T extends ToXContent> extends BlobStoreForm
}
/**
* Writes blob in atomic manner with resolving the blob name using {@link #blobName} and {@link #tempBlobName} methods.
* Writes blob in atomic manner with resolving the blob name using {@link #blobName} method.
* <p>
* The blob will be compressed and checksum will be written if required.
*
@ -131,20 +130,12 @@ public class ChecksumBlobStoreFormat<T extends ToXContent> extends BlobStoreForm
* @param name blob name
*/
public void writeAtomic(T obj, BlobContainer blobContainer, String name) throws IOException {
String blobName = blobName(name);
String tempBlobName = tempBlobName(name);
writeBlob(obj, blobContainer, tempBlobName);
try {
blobContainer.move(tempBlobName, blobName);
} catch (IOException ex) {
// Move failed - try cleaning up
try {
blobContainer.deleteBlob(tempBlobName);
} catch (Exception e) {
ex.addSuppressed(e);
final String blobName = blobName(name);
writeTo(obj, blobName, bytesArray -> {
try (InputStream stream = bytesArray.streamInput()) {
blobContainer.writeBlobAtomic(blobName, stream, bytesArray.length());
}
throw ex;
}
});
}
/**
@ -157,51 +148,35 @@ public class ChecksumBlobStoreFormat<T extends ToXContent> extends BlobStoreForm
* @param name blob name
*/
public void write(T obj, BlobContainer blobContainer, String name) throws IOException {
String blobName = blobName(name);
writeBlob(obj, blobContainer, blobName);
final String blobName = blobName(name);
writeTo(obj, blobName, bytesArray -> {
try (InputStream stream = bytesArray.streamInput()) {
blobContainer.writeBlob(blobName, stream, bytesArray.length());
}
});
}
/**
* Writes blob in atomic manner without resolving the blobName using using {@link #blobName} method.
* <p>
* The blob will be compressed and checksum will be written if required.
*
* @param obj object to be serialized
* @param blobContainer blob container
* @param blobName blob name
*/
protected void writeBlob(T obj, BlobContainer blobContainer, String blobName) throws IOException {
BytesReference bytes = write(obj);
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
private void writeTo(final T obj, final String blobName, final CheckedConsumer<BytesArray, IOException> consumer) throws IOException {
final BytesReference bytes = write(obj);
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
final String resourceDesc = "ChecksumBlobStoreFormat.writeBlob(blob=\"" + blobName + "\")";
try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, blobName, byteArrayOutputStream, BUFFER_SIZE)) {
try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, blobName, outputStream, BUFFER_SIZE)) {
CodecUtil.writeHeader(indexOutput, codec, VERSION);
try (OutputStream indexOutputOutputStream = new IndexOutputOutputStream(indexOutput) {
@Override
public void close() throws IOException {
// this is important since some of the XContentBuilders write bytes on close.
// in order to write the footer we need to prevent closing the actual index input.
} }) {
}
}) {
bytes.writeTo(indexOutputOutputStream);
}
CodecUtil.writeFooter(indexOutput);
}
BytesArray bytesArray = new BytesArray(byteArrayOutputStream.toByteArray());
try (InputStream stream = bytesArray.streamInput()) {
blobContainer.writeBlob(blobName, stream, bytesArray.length());
}
consumer.accept(new BytesArray(outputStream.toByteArray()));
}
}
/**
* Returns true if the blob is a leftover temporary blob.
*
* The temporary blobs might be left after failed atomic write operation.
*/
public boolean isTempBlobName(String blobName) {
return blobName.startsWith(ChecksumBlobStoreFormat.TEMP_FILE_PREFIX);
}
protected BytesReference write(T obj) throws IOException {
try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) {
if (compress) {
@ -222,10 +197,4 @@ public class ChecksumBlobStoreFormat<T extends ToXContent> extends BlobStoreForm
builder.endObject();
}
}
protected String tempBlobName(String name) {
return TEMP_FILE_PREFIX + String.format(Locale.ROOT, blobNameFormat, name);
}
}

45
server/src/main/java/org/elasticsearch/search/DocValueFormat.java
View File

@ -39,6 +39,7 @@ import java.text.DecimalFormatSymbols;
import java.text.NumberFormat;
import java.text.ParseException;
import java.util.Arrays;
import java.util.Base64;
import java.util.Locale;
import java.util.Objects;
import java.util.function.LongSupplier;
@ -121,6 +122,50 @@ public interface DocValueFormat extends NamedWriteable {
}
};
DocValueFormat BINARY = new DocValueFormat() {
@Override
public String getWriteableName() {
return "binary";
}
@Override
public void writeTo(StreamOutput out) throws IOException {
}
@Override
public Object format(long value) {
throw new UnsupportedOperationException();
}
@Override
public Object format(double value) {
throw new UnsupportedOperationException();
}
@Override
public String format(BytesRef value) {
return Base64.getEncoder()
.withoutPadding()
.encodeToString(Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length));
}
@Override
public long parseLong(String value, boolean roundUp, LongSupplier now) {
throw new UnsupportedOperationException();
}
@Override
public double parseDouble(String value, boolean roundUp, LongSupplier now) {
throw new UnsupportedOperationException();
}
@Override
public BytesRef parseBytesRef(String value) {
return new BytesRef(Base64.getDecoder().decode(value));
}
};
final class DateTime implements DocValueFormat {
public static final String NAME = "date_time";

1
server/src/main/java/org/elasticsearch/search/SearchModule.java
View File

@ -645,6 +645,7 @@ public class SearchModule {
registerValueFormat(DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH);
registerValueFormat(DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP);
registerValueFormat(DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW);
registerValueFormat(DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY);
}
/**

2
server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java
View File

@ -102,6 +102,6 @@ public class ClusterUpdateSettingsResponseTests extends AbstractStreamableXConte
public void testOldSerialisation() throws IOException {
ClusterUpdateSettingsResponse original = createTestInstance();
assertSerialization(original, VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.V_7_0_0_alpha1));
assertSerialization(original, VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.V_6_4_0));
}
}

4
server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.action.admin.indices.rollover;
import com.carrotsearch.randomizedtesting.annotations.Repeat;
import org.elasticsearch.Version;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentParser;
@ -132,6 +134,6 @@ public class RolloverResponseTests extends AbstractStreamableXContentTestCase<Ro
public void testOldSerialisation() throws IOException {
RolloverResponse original = createTestInstance();
assertSerialization(original, VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.V_7_0_0_alpha1));
assertSerialization(original, VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.V_6_4_0));
}
}

40
server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobContainerTests.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.blobstore.fs;
import org.elasticsearch.test.ESTestCase;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.startsWith;
public class FsBlobContainerTests extends ESTestCase {
public void testTempBlobName() {
final String blobName = randomAlphaOfLengthBetween(1, 20);
final String tempBlobName = FsBlobContainer.tempBlobName(blobName);
assertThat(tempBlobName, startsWith("pending-"));
assertThat(tempBlobName, containsString(blobName));
}
public void testIsTempBlobName() {
final String tempBlobName = FsBlobContainer.tempBlobName(randomAlphaOfLengthBetween(1, 20));
assertThat(FsBlobContainer.isTempBlobName(tempBlobName), is(true));
}
}

16
server/src/test/java/org/elasticsearch/common/blobstore/FsBlobStoreContainerTests.java → server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobStoreContainerTests.java
View File

@ -16,23 +16,27 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.blobstore;
package org.elasticsearch.common.blobstore.fs;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.common.blobstore.fs.FsBlobStore;
import org.elasticsearch.common.blobstore.BlobStore;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
import java.io.IOException;
import java.nio.file.Path;
@LuceneTestCase.SuppressFileSystems("ExtrasFS")
public class FsBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
protected BlobStore newBlobStore() throws IOException {
Path tempDir = createTempDir();
Settings settings = randomBoolean() ? Settings.EMPTY : Settings.builder().put("buffer_size", new ByteSizeValue(randomIntBetween(1, 100), ByteSizeUnit.KB)).build();
return new FsBlobStore(settings, tempDir);
final Settings settings;
if (randomBoolean()) {
settings = Settings.builder().put("buffer_size", new ByteSizeValue(randomIntBetween(1, 100), ByteSizeUnit.KB)).build();
} else {
settings = Settings.EMPTY;
}
return new FsBlobStore(settings, createTempDir());
}
}

17
server/src/test/java/org/elasticsearch/common/blobstore/FsBlobStoreTests.java → server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobStoreTests.java
View File

@ -16,10 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.blobstore;
package org.elasticsearch.common.blobstore.fs;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.common.blobstore.fs.FsBlobStore;
import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.BlobStore;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
@ -32,10 +34,15 @@ import java.nio.file.Path;
@LuceneTestCase.SuppressFileSystems("ExtrasFS")
public class FsBlobStoreTests extends ESBlobStoreTestCase {
protected BlobStore newBlobStore() throws IOException {
Path tempDir = createTempDir();
Settings settings = randomBoolean() ? Settings.EMPTY : Settings.builder().put("buffer_size", new ByteSizeValue(randomIntBetween(1, 100), ByteSizeUnit.KB)).build();
return new FsBlobStore(settings, tempDir);
final Settings settings;
if (randomBoolean()) {
settings = Settings.builder().put("buffer_size", new ByteSizeValue(randomIntBetween(1, 100), ByteSizeUnit.KB)).build();
} else {
settings = Settings.EMPTY;
}
return new FsBlobStore(settings, createTempDir());
}
public void testReadOnly() throws Exception {

22
server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
View File

@ -46,16 +46,13 @@ import org.elasticsearch.index.shard.IndexShardTestCase;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.junit.annotations.TestLogging;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@ -103,7 +100,7 @@ public class FlushIT extends ESIntegTestCase {
}
}
public void testSyncedFlush() throws ExecutionException, InterruptedException, IOException {
public void testSyncedFlush() throws Exception {
internalCluster().ensureAtLeastNumDataNodes(2);
prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)).get();
ensureGreen();
@ -246,16 +243,6 @@ public class FlushIT extends ESIntegTestCase {
assertThat(indexResult.getFailure(), nullValue());
}
private String syncedFlushDescription(ShardsSyncedFlushResult result) {
String detail = result.shardResponses().entrySet().stream()
.map(e -> "Shard [" + e.getKey() + "], result [" + e.getValue() + "]")
.collect(Collectors.joining(","));
return String.format(Locale.ROOT, "Total shards: [%d], failed: [%s], reason: [%s], detail: [%s]",
result.totalShards(), result.failed(), result.failureReason(), detail);
}
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/29392")
@TestLogging("_root:DEBUG,org.elasticsearch.indices.flush:TRACE")
public void testSyncedFlushSkipOutOfSyncReplicas() throws Exception {
internalCluster().ensureAtLeastNumDataNodes(between(2, 3));
final int numberOfReplicas = internalCluster().numDataNodes() - 1;
@ -281,7 +268,6 @@ public class FlushIT extends ESIntegTestCase {
indexDoc(IndexShardTestCase.getEngine(outOfSyncReplica), "extra_" + i);
}
final ShardsSyncedFlushResult partialResult = SyncedFlushUtil.attemptSyncedFlush(logger, internalCluster(), shardId);
logger.info("Partial seal: {}", syncedFlushDescription(partialResult));
assertThat(partialResult.totalShards(), equalTo(numberOfReplicas + 1));
assertThat(partialResult.successfulShards(), equalTo(numberOfReplicas));
assertThat(partialResult.shardResponses().get(outOfSyncReplica.routingEntry()).failureReason, equalTo(
@ -297,8 +283,6 @@ public class FlushIT extends ESIntegTestCase {
assertThat(fullResult.successfulShards(), equalTo(numberOfReplicas + 1));
}
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/29392")
@TestLogging("_root:DEBUG,org.elasticsearch.indices.flush:TRACE")
public void testDoNotRenewSyncedFlushWhenAllSealed() throws Exception {
internalCluster().ensureAtLeastNumDataNodes(between(2, 3));
final int numberOfReplicas = internalCluster().numDataNodes() - 1;
@ -315,11 +299,9 @@ public class FlushIT extends ESIntegTestCase {
index("test", "doc", Integer.toString(i));
}
final ShardsSyncedFlushResult firstSeal = SyncedFlushUtil.attemptSyncedFlush(logger, internalCluster(), shardId);
logger.info("First seal: {}", syncedFlushDescription(firstSeal));
assertThat(firstSeal.successfulShards(), equalTo(numberOfReplicas + 1));
// Do not renew synced-flush
final ShardsSyncedFlushResult secondSeal = SyncedFlushUtil.attemptSyncedFlush(logger, internalCluster(), shardId);
logger.info("Second seal: {}", syncedFlushDescription(secondSeal));
assertThat(secondSeal.successfulShards(), equalTo(numberOfReplicas + 1));
assertThat(secondSeal.syncId(), equalTo(firstSeal.syncId()));
// Shards were updated, renew synced flush.
@ -328,7 +310,6 @@ public class FlushIT extends ESIntegTestCase {
index("test", "doc", Integer.toString(i));
}
final ShardsSyncedFlushResult thirdSeal = SyncedFlushUtil.attemptSyncedFlush(logger, internalCluster(), shardId);
logger.info("Third seal: {}", syncedFlushDescription(thirdSeal));
assertThat(thirdSeal.successfulShards(), equalTo(numberOfReplicas + 1));
assertThat(thirdSeal.syncId(), not(equalTo(firstSeal.syncId())));
// Manually remove or change sync-id, renew synced flush.
@ -344,7 +325,6 @@ public class FlushIT extends ESIntegTestCase {
assertThat(shard.commitStats().syncId(), nullValue());
}
final ShardsSyncedFlushResult forthSeal = SyncedFlushUtil.attemptSyncedFlush(logger, internalCluster(), shardId);
logger.info("Forth seal: {}", syncedFlushDescription(forthSeal));
assertThat(forthSeal.successfulShards(), equalTo(numberOfReplicas + 1));
assertThat(forthSeal.syncId(), not(equalTo(thirdSeal.syncId())));
}

37
server/src/test/java/org/elasticsearch/indices/flush/SyncedFlushUtil.java
View File

@ -29,6 +29,9 @@ import org.elasticsearch.test.InternalTestCluster;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicReference;
import static org.elasticsearch.test.ESTestCase.assertBusy;
/** Utils for SyncedFlush */
public class SyncedFlushUtil {
@ -40,21 +43,31 @@ public class SyncedFlushUtil {
/**
* Blocking version of {@link SyncedFlushService#attemptSyncedFlush(ShardId, ActionListener)}
*/
public static ShardsSyncedFlushResult attemptSyncedFlush(Logger logger, InternalTestCluster cluster, ShardId shardId) {
public static ShardsSyncedFlushResult attemptSyncedFlush(Logger logger, InternalTestCluster cluster, ShardId shardId) throws Exception {
/*
* When the last indexing operation is completed, we will fire a global checkpoint sync.
* Since a global checkpoint sync request is a replication request, it will acquire an index
* shard permit on the primary when executing. If this happens at the same time while we are
* issuing the synced-flush, the synced-flush request will fail as it thinks there are
* in-flight operations. We can avoid such situation by continuing issuing another synced-flush
* if the synced-flush failed due to the ongoing operations on the primary.
*/
SyncedFlushService service = cluster.getInstance(SyncedFlushService.class);
logger.debug("Issue synced-flush on node [{}], shard [{}], cluster state [{}]",
service.nodeName(), shardId, cluster.clusterService(service.nodeName()).state());
LatchedListener<ShardsSyncedFlushResult> listener = new LatchedListener<>();
service.attemptSyncedFlush(shardId, listener);
try {
AtomicReference<LatchedListener<ShardsSyncedFlushResult>> listenerHolder = new AtomicReference<>();
assertBusy(() -> {
LatchedListener<ShardsSyncedFlushResult> listener = new LatchedListener<>();
listenerHolder.set(listener);
service.attemptSyncedFlush(shardId, listener);
listener.latch.await();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
if (listener.result != null && listener.result.failureReason() != null
&& listener.result.failureReason().contains("ongoing operations on primary")) {
throw new AssertionError(listener.result.failureReason()); // cause the assert busy to retry
}
});
if (listenerHolder.get().error != null) {
throw ExceptionsHelper.convertToElastic(listenerHolder.get().error);
}
if (listener.error != null) {
throw ExceptionsHelper.convertToElastic(listener.error);
}
return listener.result;
return listenerHolder.get().result;
}
public static final class LatchedListener<T> implements ActionListener<T> {

14
server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
View File

@ -44,6 +44,7 @@ public class DocValueFormatTests extends ESTestCase {
entries.add(new Entry(DocValueFormat.class, DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH));
entries.add(new Entry(DocValueFormat.class, DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP));
entries.add(new Entry(DocValueFormat.class, DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW));
entries.add(new Entry(DocValueFormat.class, DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY));
NamedWriteableRegistry registry = new NamedWriteableRegistry(entries);
BytesStreamOutput out = new BytesStreamOutput();
@ -82,6 +83,11 @@ public class DocValueFormatTests extends ESTestCase {
out.writeNamedWriteable(DocValueFormat.RAW);
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
assertSame(DocValueFormat.RAW, in.readNamedWriteable(DocValueFormat.class));
out = new BytesStreamOutput();
out.writeNamedWriteable(DocValueFormat.BINARY);
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
assertSame(DocValueFormat.BINARY, in.readNamedWriteable(DocValueFormat.class));
}
public void testRawFormat() {
@ -96,6 +102,14 @@ public class DocValueFormatTests extends ESTestCase {
assertEquals("abc", DocValueFormat.RAW.format(new BytesRef("abc")));
}
public void testBinaryFormat() {
assertEquals("", DocValueFormat.BINARY.format(new BytesRef()));
assertEquals("KmQ", DocValueFormat.BINARY.format(new BytesRef(new byte[] {42, 100})));
assertEquals(new BytesRef(), DocValueFormat.BINARY.parseBytesRef(""));
assertEquals(new BytesRef(new byte[] {42, 100}), DocValueFormat.BINARY.parseBytesRef("KmQ"));
}
public void testBooleanFormat() {
assertEquals(false, DocValueFormat.BOOLEAN.format(0));
assertEquals(true, DocValueFormat.BOOLEAN.format(1));

14
server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.search.fields;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
@ -700,7 +701,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
assertThat(fields.get("test_field").getValue(), equalTo("foobar"));
}
public void testFieldsPulledFromFieldData() throws Exception {
public void testDocValueFields() throws Exception {
createIndex("test");
String mapping = Strings
@ -744,6 +745,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
.endObject()
.startObject("binary_field")
.field("type", "binary")
.field("doc_values", true) // off by default on binary fields
.endObject()
.startObject("ip_field")
.field("type", "ip")
@ -766,6 +768,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
.field("double_field", 6.0d)
.field("date_field", Joda.forPattern("dateOptionalTime").printer().print(date))
.field("boolean_field", true)
.field("binary_field", new byte[] {42, 100})
.field("ip_field", "::1")
.endObject()).execute().actionGet();
@ -782,6 +785,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
.addDocValueField("double_field")
.addDocValueField("date_field")
.addDocValueField("boolean_field")
.addDocValueField("binary_field")
.addDocValueField("ip_field");
SearchResponse searchResponse = builder.execute().actionGet();
@ -790,7 +794,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
Set<String> fields = new HashSet<>(searchResponse.getHits().getAt(0).getFields().keySet());
assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field",
"float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field",
"ip_field")));
"binary_field", "ip_field")));
assertThat(searchResponse.getHits().getAt(0).getFields().get("byte_field").getValue().toString(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("short_field").getValue().toString(), equalTo("2"));
@ -802,6 +806,8 @@ public class SearchFieldsIT extends ESIntegTestCase {
assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true));
assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(),
equalTo(new BytesRef(new byte[] {42, 100})));
assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));
builder = client().prepareSearch().setQuery(matchAllQuery())
@ -815,6 +821,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
.addDocValueField("double_field", "use_field_mapping")
.addDocValueField("date_field", "use_field_mapping")
.addDocValueField("boolean_field", "use_field_mapping")
.addDocValueField("binary_field", "use_field_mapping")
.addDocValueField("ip_field", "use_field_mapping");
searchResponse = builder.execute().actionGet();
@ -823,7 +830,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
fields = new HashSet<>(searchResponse.getHits().getAt(0).getFields().keySet());
assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field",
"float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field",
"ip_field")));
"binary_field", "ip_field")));
assertThat(searchResponse.getHits().getAt(0).getFields().get("byte_field").getValue().toString(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("short_field").getValue().toString(), equalTo("2"));
@ -836,6 +843,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true));
assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));
builder = client().prepareSearch().setQuery(matchAllQuery())

2
server/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
View File

@ -100,7 +100,7 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase {
}
Thread.sleep(100);
}
fail("Timeout!!!");
fail("Timeout waiting for node [" + node + "] to be blocked");
}
public SnapshotInfo waitForCompletion(String repository, String snapshotName, TimeValue timeout) throws InterruptedException {

42
server/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
View File

@ -224,52 +224,16 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
IOException writeBlobException = expectThrows(IOException.class, () -> {
BlobContainer wrapper = new BlobContainerWrapper(blobContainer) {
@Override
public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException {
throw new IOException("Exception thrown in writeBlob() for " + blobName);
public void writeBlobAtomic(String blobName, InputStream inputStream, long blobSize) throws IOException {
throw new IOException("Exception thrown in writeBlobAtomic() for " + blobName);
}
};
checksumFormat.writeAtomic(blobObj, wrapper, name);
});
assertEquals("Exception thrown in writeBlob() for pending-" + name, writeBlobException.getMessage());
assertEquals("Exception thrown in writeBlobAtomic() for " + name, writeBlobException.getMessage());
assertEquals(0, writeBlobException.getSuppressed().length);
}
{
IOException moveException = expectThrows(IOException.class, () -> {
BlobContainer wrapper = new BlobContainerWrapper(blobContainer) {
@Override
public void move(String sourceBlobName, String targetBlobName) throws IOException {
throw new IOException("Exception thrown in move() for " + sourceBlobName);
}
};
checksumFormat.writeAtomic(blobObj, wrapper, name);
});
assertEquals("Exception thrown in move() for pending-" + name, moveException.getMessage());
assertEquals(0, moveException.getSuppressed().length);
}
{
IOException moveThenDeleteException = expectThrows(IOException.class, () -> {
BlobContainer wrapper = new BlobContainerWrapper(blobContainer) {
@Override
public void move(String sourceBlobName, String targetBlobName) throws IOException {
throw new IOException("Exception thrown in move() for " + sourceBlobName);
}
@Override
public void deleteBlob(String blobName) throws IOException {
throw new IOException("Exception thrown in deleteBlob() for " + blobName);
}
};
checksumFormat.writeAtomic(blobObj, wrapper, name);
});
assertEquals("Exception thrown in move() for pending-" + name, moveThenDeleteException.getMessage());
assertEquals(1, moveThenDeleteException.getSuppressed().length);
final Throwable suppressedThrowable = moveThenDeleteException.getSuppressed()[0];
assertTrue(suppressedThrowable instanceof IOException);
assertEquals("Exception thrown in deleteBlob() for pending-" + name, suppressedThrowable.getMessage());
}
}
protected BlobStore createTestBlobStore() throws IOException {

10
server/src/test/java/org/elasticsearch/snapshots/mockstore/BlobContainerWrapper.java
View File

@ -53,11 +53,21 @@ public class BlobContainerWrapper implements BlobContainer {
delegate.writeBlob(blobName, inputStream, blobSize);
}
@Override
public void writeBlobAtomic(final String blobName, final InputStream inputStream, final long blobSize) throws IOException {
delegate.writeBlobAtomic(blobName, inputStream, blobSize);
}
@Override
public void deleteBlob(String blobName) throws IOException {
delegate.deleteBlob(blobName);
}
@Override
public void deleteBlobIgnoringIfNotExists(final String blobName) throws IOException {
delegate.deleteBlobIgnoringIfNotExists(blobName);
}
@Override
public Map<String, BlobMetaData> listBlobs() throws IOException {
return delegate.listBlobs();

75
server/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
View File

@ -19,6 +19,28 @@
package org.elasticsearch.snapshots.mockstore;
import com.carrotsearch.randomizedtesting.RandomizedContext;
import org.apache.lucene.index.CorruptIndexException;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.blobstore.BlobMetaData;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.BlobStore;
import org.elasticsearch.common.blobstore.fs.FsBlobContainer;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.env.Environment;
import org.elasticsearch.plugins.RepositoryPlugin;
import org.elasticsearch.repositories.IndexId;
import org.elasticsearch.repositories.Repository;
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.snapshots.SnapshotId;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
@ -29,31 +51,11 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicLong;
import com.carrotsearch.randomizedtesting.RandomizedContext;
import org.apache.lucene.index.CorruptIndexException;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.blobstore.BlobMetaData;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.BlobStore;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.env.Environment;
import org.elasticsearch.plugins.RepositoryPlugin;
import org.elasticsearch.repositories.Repository;
import org.elasticsearch.repositories.IndexId;
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.snapshots.SnapshotId;
public class MockRepository extends FsRepository {
public static class Plugin extends org.elasticsearch.plugins.Plugin implements RepositoryPlugin {
@ -325,6 +327,12 @@ public class MockRepository extends FsRepository {
super.deleteBlob(blobName);
}
@Override
public void deleteBlobIgnoringIfNotExists(String blobName) throws IOException {
maybeIOExceptionOrBlock(blobName);
super.deleteBlobIgnoringIfNotExists(blobName);
}
@Override
public Map<String, BlobMetaData> listBlobs() throws IOException {
maybeIOExceptionOrBlock("");
@ -365,6 +373,31 @@ public class MockRepository extends FsRepository {
maybeIOExceptionOrBlock(blobName);
}
}
@Override
public void writeBlobAtomic(final String blobName, final InputStream inputStream, final long blobSize) throws IOException {
final Random random = RandomizedContext.current().getRandom();
if (random.nextBoolean()) {
if ((delegate() instanceof FsBlobContainer) && (random.nextBoolean())) {
// Simulate a failure between the write and move operation in FsBlobContainer
final String tempBlobName = FsBlobContainer.tempBlobName(blobName);
super.writeBlob(tempBlobName, inputStream, blobSize);
maybeIOExceptionOrBlock(blobName);
final FsBlobContainer fsBlobContainer = (FsBlobContainer) delegate();
fsBlobContainer.move(tempBlobName, blobName);
} else {
// Atomic write since it is potentially supported
// by the delegating blob container
maybeIOExceptionOrBlock(blobName);
super.writeBlobAtomic(blobName, inputStream, blobSize);
}
} else {
// Simulate a non-atomic write since many blob container
// implementations does not support atomic write
maybeIOExceptionOrBlock(blobName);
super.writeBlob(blobName, inputStream, blobSize);
}
}
}
}
}

6
test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
View File

@ -158,7 +158,11 @@ public abstract class ESBlobStoreContainerTestCase extends ESTestCase {
protected void writeBlob(final BlobContainer container, final String blobName, final BytesArray bytesArray) throws IOException {
try (InputStream stream = bytesArray.streamInput()) {
container.writeBlob(blobName, stream, bytesArray.length());
if (randomBoolean()) {
container.writeBlob(blobName, stream, bytesArray.length());
} else {
container.writeBlobAtomic(blobName, stream, bytesArray.length());
}
}
}

3
x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
View File

@ -411,7 +411,8 @@ public class LicenseService extends AbstractLifecycleComponent implements Cluste
// auto-generate license if no licenses ever existed or if the current license is basic and
// needs extended or if the license signature needs to be updated. this will trigger a subsequent cluster changed event
if (currentClusterState.getNodes().isLocalNodeElectedMaster() &&
(noLicense || LicenseUtils.licenseNeedsExtended(currentLicense) || LicenseUtils.signatureNeedsUpdate(currentLicense))) {
(noLicense || LicenseUtils.licenseNeedsExtended(currentLicense) ||
LicenseUtils.signatureNeedsUpdate(currentLicense, currentClusterState.nodes()))) {
registerOrUpdateSelfGeneratedLicense();
}
} else if (logger.isDebugEnabled()) {

25
x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseUtils.java
View File

@ -6,8 +6,12 @@
package org.elasticsearch.license;
import org.elasticsearch.ElasticsearchSecurityException;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.rest.RestStatus;
import java.util.stream.StreamSupport;
public class LicenseUtils {
public static final String EXPIRED_FEATURE_METADATA = "es.license.expired.feature";
@ -42,8 +46,25 @@ public class LicenseUtils {
* Checks if the signature of a self generated license with older version needs to be
* recreated with the new key
*/
public static boolean signatureNeedsUpdate(License license) {
public static boolean signatureNeedsUpdate(License license, DiscoveryNodes currentNodes) {
assert License.VERSION_CRYPTO_ALGORITHMS == License.VERSION_CURRENT : "update this method when adding a new version";
return ("basic".equals(license.type()) || "trial".equals(license.type())) &&
(license.version() < License.VERSION_CRYPTO_ALGORITHMS);
// only upgrade signature when all nodes are ready to deserialize the new signature
(license.version() < License.VERSION_CRYPTO_ALGORITHMS &&
compatibleLicenseVersion(currentNodes) == License.VERSION_CRYPTO_ALGORITHMS
);
}
public static int compatibleLicenseVersion(DiscoveryNodes currentNodes) {
assert License.VERSION_CRYPTO_ALGORITHMS == License.VERSION_CURRENT : "update this method when adding a new version";
if (StreamSupport.stream(currentNodes.spliterator(), false)
.allMatch(node -> node.getVersion().onOrAfter(Version.V_6_4_0))) {
// License.VERSION_CRYPTO_ALGORITHMS was introduced in 6.4.0
return License.VERSION_CRYPTO_ALGORITHMS;
} else {
return License.VERSION_START_DATE;
}
}
}

5
x-pack/plugin/core/src/main/java/org/elasticsearch/license/SelfGeneratedLicense.java
View File

@ -5,6 +5,7 @@
*/
package org.elasticsearch.license;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@ -26,8 +27,8 @@ import static org.elasticsearch.license.CryptUtils.decrypt;
class SelfGeneratedLicense {
public static License create(License.Builder specBuilder) {
return create(specBuilder, License.VERSION_CURRENT);
public static License create(License.Builder specBuilder, DiscoveryNodes currentNodes) {
return create(specBuilder, LicenseUtils.compatibleLicenseVersion(currentNodes));
}
public static License create(License.Builder specBuilder, int version) {

2
x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartBasicClusterTask.java
View File

@ -73,7 +73,7 @@ public class StartBasicClusterTask extends ClusterStateUpdateTask {
.issueDate(issueDate)
.type("basic")
.expiryDate(LicenseService.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS);
License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder);
License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder, currentState.nodes());
if (request.isAcknowledged() == false && currentLicense != null) {
Map<String, String[]> ackMessages = LicenseService.getAckMessages(selfGeneratedLicense, currentLicense);
if (ackMessages.isEmpty() == false) {

2
x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartTrialClusterTask.java
View File

@ -82,7 +82,7 @@ public class StartTrialClusterTask extends ClusterStateUpdateTask {
.issueDate(issueDate)
.type(request.getType())
.expiryDate(expiryDate);
License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder);
License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder, currentState.nodes());
LicensesMetaData newLicensesMetaData = new LicensesMetaData(selfGeneratedLicense, Version.CURRENT);
mdBuilder.putCustom(LicensesMetaData.TYPE, newLicensesMetaData);
return ClusterState.builder(currentState).metaData(mdBuilder).build();

8
x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartupSelfGeneratedLicenseTask.java
View File

@ -61,7 +61,7 @@ public class StartupSelfGeneratedLicenseTask extends ClusterStateUpdateTask {
"]. Must be trial or basic.");
}
return updateWithLicense(currentState, type);
} else if (LicenseUtils.signatureNeedsUpdate(currentLicensesMetaData.getLicense())) {
} else if (LicenseUtils.signatureNeedsUpdate(currentLicensesMetaData.getLicense(), currentState.nodes())) {
return updateLicenseSignature(currentState, currentLicensesMetaData);
} else if (LicenseUtils.licenseNeedsExtended(currentLicensesMetaData.getLicense())) {
return extendBasic(currentState, currentLicensesMetaData);
@ -87,7 +87,7 @@ public class StartupSelfGeneratedLicenseTask extends ClusterStateUpdateTask {
.issueDate(issueDate)
.type(type)
.expiryDate(expiryDate);
License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder);
License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder, currentState.nodes());
Version trialVersion = currentLicenseMetaData.getMostRecentTrialVersion();
LicensesMetaData newLicenseMetadata = new LicensesMetaData(selfGeneratedLicense, trialVersion);
mdBuilder.putCustom(LicensesMetaData.TYPE, newLicenseMetadata);
@ -120,7 +120,7 @@ public class StartupSelfGeneratedLicenseTask extends ClusterStateUpdateTask {
.issueDate(currentLicense.issueDate())
.type("basic")
.expiryDate(LicenseService.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS);
License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder);
License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder, currentLicense.version());
Version trialVersion = currentLicenseMetadata.getMostRecentTrialVersion();
return new LicensesMetaData(selfGeneratedLicense, trialVersion);
}
@ -141,7 +141,7 @@ public class StartupSelfGeneratedLicenseTask extends ClusterStateUpdateTask {
.issueDate(issueDate)
.type(type)
.expiryDate(expiryDate);
License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder);
License selfGeneratedLicense = SelfGeneratedLicense.create(specBuilder, currentState.nodes());
LicensesMetaData licensesMetaData;
if ("trial".equals(type)) {
licensesMetaData = new LicensesMetaData(selfGeneratedLicense, Version.CURRENT);

4
x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseRegistrationTests.java
View File

@ -104,7 +104,7 @@ public class LicenseRegistrationTests extends AbstractLicenseServiceTestCase {
.issueDate(dateMath("now-10h", now))
.type("basic")
.expiryDate(dateMath("now-2h", now));
License license = SelfGeneratedLicense.create(builder);
License license = SelfGeneratedLicense.create(builder, License.VERSION_CURRENT);
XPackLicenseState licenseState = new XPackLicenseState(Settings.EMPTY);
setInitialState(license, licenseState, Settings.EMPTY);
@ -125,4 +125,4 @@ public class LicenseRegistrationTests extends AbstractLicenseServiceTestCase {
assertEquals(LicenseService.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS, licenseMetaData.getLicense().expiryDate());
assertEquals(uid, licenseMetaData.getLicense().uid());
}
}
}

2
x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseSerializationTests.java
View File

@ -111,7 +111,7 @@ public class LicenseSerializationTests extends ESTestCase {
.issueDate(now)
.type("basic")
.expiryDate(LicenseService.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS);
License license = SelfGeneratedLicense.create(specBuilder);
License license = SelfGeneratedLicense.create(specBuilder, License.VERSION_CURRENT);
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
license.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap(License.REST_VIEW_MODE, "true")));
builder.flush();

2
x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetaDataSerializationTests.java
View File

@ -95,7 +95,7 @@ public class LicensesMetaDataSerializationTests extends ESTestCase {
.issueDate(issueDate)
.type(randomBoolean() ? "trial" : "basic")
.expiryDate(issueDate + TimeValue.timeValueHours(2).getMillis());
final License trialLicense = SelfGeneratedLicense.create(specBuilder);
final License trialLicense = SelfGeneratedLicense.create(specBuilder, License.VERSION_CURRENT);
LicensesMetaData licensesMetaData = new LicensesMetaData(trialLicense, Version.CURRENT);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();

7
x-pack/plugin/core/src/test/java/org/elasticsearch/license/SelfGeneratedLicenseTests.java
View File

@ -34,7 +34,7 @@ public class SelfGeneratedLicenseTests extends ESTestCase {
.type(randomBoolean() ? "trial" : "basic")
.issueDate(issueDate)
.expiryDate(issueDate + TimeValue.timeValueHours(2).getMillis());
License trialLicense = SelfGeneratedLicense.create(specBuilder);
License trialLicense = SelfGeneratedLicense.create(specBuilder, License.VERSION_CURRENT);
assertThat(SelfGeneratedLicense.verify(trialLicense), equalTo(true));
}
@ -47,7 +47,7 @@ public class SelfGeneratedLicenseTests extends ESTestCase {
.maxNodes(5)
.issueDate(issueDate)
.expiryDate(issueDate + TimeValue.timeValueHours(2).getMillis());
License trialLicense = SelfGeneratedLicense.create(specBuilder);
License trialLicense = SelfGeneratedLicense.create(specBuilder, License.VERSION_CURRENT);
final String originalSignature = trialLicense.signature();
License tamperedLicense = License.builder().fromLicenseSpec(trialLicense, originalSignature)
.expiryDate(System.currentTimeMillis() + TimeValue.timeValueHours(5).getMillis())
@ -70,7 +70,8 @@ public class SelfGeneratedLicenseTests extends ESTestCase {
.issueDate(issueDate)
.expiryDate(issueDate + TimeValue.timeValueHours(2).getMillis());
License pre20TrialLicense = specBuilder.build();
License license = SelfGeneratedLicense.create(License.builder().fromPre20LicenseSpec(pre20TrialLicense).type("trial"));
License license = SelfGeneratedLicense.create(License.builder().fromPre20LicenseSpec(pre20TrialLicense).type("trial"),
License.VERSION_CURRENT);
assertThat(SelfGeneratedLicense.verify(license), equalTo(true));
}

17
x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
View File

@ -7,7 +7,6 @@ package org.elasticsearch.xpack.security;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
@ -17,7 +16,6 @@ import org.elasticsearch.action.support.DestructiveOperations;
import org.elasticsearch.bootstrap.BootstrapCheck;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
@ -112,7 +110,6 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationServiceField;
import org.elasticsearch.xpack.core.security.authc.DefaultAuthenticationFailureHandler;
import org.elasticsearch.xpack.core.security.authc.Realm;
import org.elasticsearch.xpack.core.security.authc.RealmSettings;
import org.elasticsearch.xpack.core.security.authc.TokenMetaData;
import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
@ -934,7 +931,8 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
if (enabled) {
return new ValidateTLSOnJoin(XPackSettings.TRANSPORT_SSL_ENABLED.get(settings),
DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings))
.andThen(new ValidateUpgradedSecurityIndex());
.andThen(new ValidateUpgradedSecurityIndex())
.andThen(new ValidateLicenseCanBeDeserialized());
}
return null;
}
@ -971,6 +969,17 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
}
}
static final class ValidateLicenseCanBeDeserialized implements BiConsumer<DiscoveryNode, ClusterState> {
@Override
public void accept(DiscoveryNode node, ClusterState state) {
License license = LicenseService.getLicense(state.metaData());
if (license != null && license.version() >= License.VERSION_CRYPTO_ALGORITHMS && node.getVersion().before(Version.V_6_4_0)) {
throw new IllegalStateException("node " + node + " is on version [" + node.getVersion() +
"] that cannot deserialize the license format [" + license.version() + "], upgrade node to at least 6.4.0");
}
}
}
@Override
public void reloadSPI(ClassLoader loader) {
securityExtensions.addAll(SecurityExtension.loadExtensions(loader));

16
x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
View File

@ -28,6 +28,7 @@ import org.elasticsearch.license.TestUtils;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.plugins.MapperPlugin;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.VersionUtils;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.watcher.ResourceWatcherService;
import org.elasticsearch.xpack.core.XPackSettings;
@ -278,6 +279,19 @@ public class SecurityTests extends ESTestCase {
}
}
public void testJoinValidatorForLicenseDeserialization() throws Exception {
DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(),
VersionUtils.randomVersionBetween(random(), null, Version.V_6_3_0));
MetaData.Builder builder = MetaData.builder();
License license = TestUtils.generateSignedLicense(null,
randomIntBetween(License.VERSION_CRYPTO_ALGORITHMS, License.VERSION_CURRENT), -1, TimeValue.timeValueHours(24));
TestUtils.putLicense(builder, license);
ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(builder.build()).build();
IllegalStateException e = expectThrows(IllegalStateException.class,
() -> new Security.ValidateLicenseCanBeDeserialized().accept(node, state));
assertThat(e.getMessage(), containsString("cannot deserialize the license format"));
}
public void testIndexJoinValidator_Old_And_Rolling() throws Exception {
createComponents(Settings.EMPTY);
BiConsumer<DiscoveryNode, ClusterState> joinValidator = security.getJoinValidator();
@ -345,7 +359,7 @@ public class SecurityTests extends ESTestCase {
.nodes(discoveryNodes).build();
joinValidator.accept(node, clusterState);
}
public void testGetFieldFilterSecurityEnabled() throws Exception {
createComponents(Settings.EMPTY);
Function<String, Predicate<String>> fieldFilter = security.getFieldFilter();