From ecf53b167e27cceae0ea131d4db440ea5a526e0c Mon Sep 17 00:00:00 2001 From: Clinton Gormley Date: Fri, 5 Jun 2015 17:50:10 +0200 Subject: [PATCH 1/5] Docs: Added explanation of when to use the upgrade API Closes #9779 --- docs/reference/indices/upgrade.asciidoc | 61 +++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/docs/reference/indices/upgrade.asciidoc b/docs/reference/indices/upgrade.asciidoc index 046dde2fe8a..bb6747daf09 100644 --- a/docs/reference/indices/upgrade.asciidoc +++ b/docs/reference/indices/upgrade.asciidoc @@ -1,9 +1,64 @@ [[indices-upgrade]] == Upgrade -The upgrade API allows to upgrade one or more indices to the latest format -through an API. The upgrade process converts any segments written -with previous formats. +The upgrade API allows to upgrade one or more indices to the latest Lucene +format through an API. The upgrade process converts any segments written with +older formats. + +.When to use the `upgrade` API +************************************************** + +Newer versions of Lucene often come with a new index format which provides bug +fixes and performance improvements. In order to take advantage of these +improvements, the segments in each shard need to be rewritten using the latest +Lucene format. + +.Automatic upgrading + +Indices that are actively being written to will automatically write new +segments in the latest format. The background merge process which combines +multiple small segments into a single bigger segment will also write the new +merged segment in the latest format. + +.Optional manual upgrades + +Some old segments may never be merged away because they are already too big to +be worth merging, and indices that no longer receive changes will not be +upgraded automatically. Upgrading segments is not required for most +Elasticsearch upgrades because it can read older formats from the current and +previous major version of Lucene. + +You can, however, choose to upgrade old segments manually to take advantage of +the latest format. The `upgrade` API will rewrite any old segments in the +latest Lucene format. It can be run on one index, multiple or all indices, so +you can control when it is run and how many indices it should upgrade. + +.When you must use the `upgrade` API + +Elasticsearch can only read formats from the current and previous major +version of Lucene. For instance, Elasticsearch 2.x (Lucene 5) can read disk +formats from Elasticsearch 0.90 and 1.x (Lucene 4), but not from Elasticsearch +0.20 and before (Lucene 3). + +In fact, an Elasticsearch 2.0 cluster will refuse to start if any indices +created before Elasticsearch 0.90 are present, and it will refuse to open them +if they are imported as dangling indices later on. It will not be possible to +restore an index created with Elasticsearch 0.20.x and before into a 2.0 +cluster. + +These ancient indices must either be deleted or upgraded before migrating to +Elasticsearch 2.0. Upgrading will: + +* Rewrite old segments in the latest Lucene format. +* Add the `index.version.minimum_compatible` setting to the index, to mark it as + 2.0 compatible coming[1.6.0]. + +Instead of upgrading all segments that weren't written with the most recent +version of Lucene, you can choose to do the minimum work required before +moving to Elasticsearch 2.0, by specifying the `only_ancient_segments` option, +which will only rewrite segments written by Lucene 3. + +************************************************** [float] === Start an upgrade From 02c9c8998e0e48a5855e81646514e3d4cf60dfa7 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 5 Jun 2015 19:17:21 +0200 Subject: [PATCH 2/5] [maven] update maven-assembly-plugin to 2.5.5 ```xml org.apache.maven.plugins maven-assembly-plugin 2.5.5 ``` ## Release Notes - Maven Assembly Plugin - Version 2.5.5 * Bug - [MASSEMBLY-767] - Schema missing from the web site - [MASSEMBLY-768] - JarInputStream unable to find manifest created by version 2.5.4 - [MASSEMBLY-769] - ZIP fileMode permissions not properly set with dependencySet and unpackOptions --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 764444c2741..b5aaff311a0 100644 --- a/pom.xml +++ b/pom.xml @@ -507,7 +507,7 @@ org.apache.maven.plugins maven-assembly-plugin - 2.5.4 + 2.5.5 org.apache.maven.plugins From 1d02212b1c87a1e296157a526f02be8afd02dd3a Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 5 Jun 2015 06:57:03 -1000 Subject: [PATCH 3/5] Snapshot/Restore: blob store shouldn't try deleting the write.lock file at the end of the restore process Since we are creating write.lock earlier now, blob store shouldn't attempt deleting this file during clean up at the end of the restore process. The file is locked and the blog store doesn't succeed, but it generates a lot of useless warnings "failed to delete file [write.lock] during snapshot cleanup". Closes #11517 --- .../BlobStoreIndexShardRepository.java | 22 ++++++++----------- .../org/elasticsearch/index/store/Store.java | 12 ++++++++-- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java index 572cda21d09..fae398e1a90 100644 --- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java +++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java @@ -19,15 +19,10 @@ package org.elasticsearch.index.snapshots.blobstore; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.io.ByteStreams; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexFormatTooNewException; -import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.*; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; @@ -902,13 +897,14 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements /// now, go over and clean files that are in the store, but were not in the snapshot try { for (String storeFile : store.directory().listAll()) { - if (!Store.isChecksum(storeFile) && !snapshotFiles.containPhysicalIndexFile(storeFile)) { - try { - store.deleteQuiet("restore", storeFile); - store.directory().deleteFile(storeFile); - } catch (IOException e) { - logger.warn("[{}] failed to delete file [{}] during snapshot cleanup", snapshotId, storeFile); - } + if (Store.isAutogenerated(storeFile) || snapshotFiles.containPhysicalIndexFile(storeFile)) { + continue; //skip write.lock, checksum files and files that exist in the snapshot + } + try { + store.deleteQuiet("restore", storeFile); + store.directory().deleteFile(storeFile); + } catch (IOException e) { + logger.warn("[{}] failed to delete file [{}] during snapshot cleanup", snapshotId, storeFile); } } } catch (IOException e) { diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java index d92128a319c..2617e95b418 100644 --- a/core/src/main/java/org/elasticsearch/index/store/Store.java +++ b/core/src/main/java/org/elasticsearch/index/store/Store.java @@ -586,7 +586,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref try (Lock writeLock = Lucene.acquireWriteLock(directory)) { final StoreDirectory dir = directory; for (String existingFile : dir.listAll()) { - if (existingFile.equals(IndexWriter.WRITE_LOCK_NAME) || Store.isChecksum(existingFile) || sourceMetaData.contains(existingFile)) { + if (Store.isAutogenerated(existingFile) || sourceMetaData.contains(existingFile)) { continue; // don't delete snapshot file, or the checksums file (note, this is extra protection since the Store won't delete checksum) } try { @@ -1206,11 +1206,19 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref public static final String CHECKSUMS_PREFIX = "_checksums-"; - public static final boolean isChecksum(String name) { + public static boolean isChecksum(String name) { // TODO can we drowp .cks return name.startsWith(CHECKSUMS_PREFIX) || name.endsWith(".cks"); // bwcomapt - .cks used to be a previous checksum file } + /** + * Returns true if the file is auto-generated by the store and shouldn't be deleted during cleanup. + * This includes write lock and checksum files + */ + public static boolean isAutogenerated(String name) { + return IndexWriter.WRITE_LOCK_NAME.equals(name) || isChecksum(name); + } + /** * Produces a string representation of the given digest value. */ From 6aa27a16c6684354654504e8f9c7b42c84d85398 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Fri, 5 Jun 2015 21:21:09 +0200 Subject: [PATCH 4/5] GatewayAllocator: reset rerouting flag after error After asynchronously fetching shard information the gateway allocator issues a reroute via a cluster state update task. #11421 introduced an optimization trying to avoid submitting unneeded reroutes when results for many shards come in together. This is done by having a rerouting flag, indicating a pending reroute is coming and thus any new incoming shard info doesn't need to issue a reroute. This flag wasn't reset upon an error in the reroute update task. Most notably - if a master node had to step during to a min_master_node violation, it could reject an ongoing reroute. Lacking to reset the flag causing it to skip any future reroute, when the node became master again. Closes #11519 --- .../main/java/org/elasticsearch/gateway/GatewayAllocator.java | 1 + .../java/org/elasticsearch/cluster/MinimumMasterNodesTests.java | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java b/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java index e0e7a847d45..f59e271c04f 100644 --- a/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java +++ b/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java @@ -544,6 +544,7 @@ public class GatewayAllocator extends AbstractComponent { @Override public void onFailure(String source, Throwable t) { + rerouting.set(false); logger.warn("failed to perform reroute post async fetch for {}", t, source); } }); diff --git a/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesTests.java b/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesTests.java index 2034dc41b81..68d682ddf7b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesTests.java @@ -31,7 +31,6 @@ import org.elasticsearch.discovery.zen.elect.ElectMasterService; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; -import org.elasticsearch.test.junit.annotations.TestLogging; import org.junit.Test; import java.util.concurrent.ExecutionException; @@ -164,7 +163,6 @@ public class MinimumMasterNodesTests extends ElasticsearchIntegrationTest { } @Test @Slow - @TestLogging("cluster.routing.allocation.allocator:TRACE") public void multipleNodesShutdownNonMasterNodes() throws Exception { Settings settings = settingsBuilder() .put("discovery.type", "zen") From 4c981ff4bfc250080d521af105b5e8589c9fc517 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Fri, 5 Jun 2015 16:13:09 +0200 Subject: [PATCH 5/5] [BUILD] Don't shade core artifacts This commit adds an additioal jar that is shaded and keeps all the artifacts that are used by default on the server-side unshaded. Users that need a shaded jar can now use the `shaded` classifyer to pull the shaded minimized jar in instead. Including the shaded jar in a downstream project looks like this: ```XML org.elasticsearch elasticsearch shaded ``` --- core/pom.xml | 7 ++++--- core/src/main/assemblies/common-bin.xml | 14 ++++++++++++++ pom.xml | 4 ---- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index f6bdebdd4d3..426d4c4d125 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -18,8 +18,6 @@ - - unshaded /usr/share/elasticsearch /usr/share/elasticsearch/bin @@ -415,9 +413,12 @@ - true + true + shaded + false true true + false com.google.guava:guava diff --git a/core/src/main/assemblies/common-bin.xml b/core/src/main/assemblies/common-bin.xml index 49a00eab4ee..011a9c8b2f1 100644 --- a/core/src/main/assemblies/common-bin.xml +++ b/core/src/main/assemblies/common-bin.xml @@ -11,6 +11,20 @@ com.spatial4j:spatial4j com.vividsolutions:jts org.codehaus.groovy:groovy-all + com.google.guava:guava + com.carrotsearch:hppc + com.fasterxml.jackson.core:jackson-core + com.fasterxml.jackson.dataformat:jackson-dataformat-smile + com.fasterxml.jackson.dataformat:jackson-dataformat-yaml + com.fasterxml.jackson.dataformat:jackson-dataformat-cbor + joda-time:joda-time + org.joda:joda-convert + io.netty:netty + com.ning:compress-lzf + com.github.spullara.mustache.java:compiler + com.tdunning:t-digest + org.apache.commons:commons-lang3 + commons-cli:commons-cli diff --git a/pom.xml b/pom.xml index b5aaff311a0..af5fc845ece 100644 --- a/pom.xml +++ b/pom.xml @@ -760,8 +760,6 @@ true false - - false jsr166e/** @@ -792,8 +790,6 @@ true false - - false jdk-unsafe