From ba291605626594b6bf0ea06b8061547aa4231799 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 8 Dec 2011 12:07:55 +0200 Subject: [PATCH 001/270] allow to enabled/disable gc deletes internally --- .../org/elasticsearch/index/engine/Engine.java | 2 ++ .../index/engine/robin/RobinEngine.java | 15 +++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/engine/Engine.java b/src/main/java/org/elasticsearch/index/engine/Engine.java index 1fef90482dd..37f358cf7a9 100644 --- a/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -56,6 +56,8 @@ public interface Engine extends IndexShardComponent, CloseableComponent { */ TimeValue defaultRefreshInterval(); + void enableGcDeletes(boolean enableGcDeletes); + void updateIndexingBufferSize(ByteSizeValue indexingBufferSize); void addFailedEngineListener(FailedEngineListener listener); diff --git a/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java b/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java index 334782e32cc..2a285d33c73 100644 --- a/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java +++ b/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java @@ -92,6 +92,8 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine { private long gcDeletesInMillis; + private volatile boolean enableGcDeletes = true; + private final ThreadPool threadPool; private final IndexSettingsService indexSettingsService; @@ -286,6 +288,11 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine { return new TimeValue(1, TimeUnit.SECONDS); } + @Override + public void enableGcDeletes(boolean enableGcDeletes) { + this.enableGcDeletes = enableGcDeletes; + } + public GetResult get(Get get) throws EngineException { rwl.readLock().lock(); try { @@ -386,7 +393,7 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine { if (versionValue == null) { currentVersion = loadCurrentVersionFromIndex(create.uid()); } else { - if (versionValue.delete() && (threadPool.estimatedTimeInMillis() - versionValue.time()) > gcDeletesInMillis) { + if (enableGcDeletes && versionValue.delete() && (threadPool.estimatedTimeInMillis() - versionValue.time()) > gcDeletesInMillis) { currentVersion = -1; // deleted, and GC } else { currentVersion = versionValue.version(); @@ -510,7 +517,7 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine { if (versionValue == null) { currentVersion = loadCurrentVersionFromIndex(index.uid()); } else { - if (versionValue.delete() && (threadPool.estimatedTimeInMillis() - versionValue.time()) > gcDeletesInMillis) { + if (enableGcDeletes && versionValue.delete() && (threadPool.estimatedTimeInMillis() - versionValue.time()) > gcDeletesInMillis) { currentVersion = -1; // deleted, and GC } else { currentVersion = versionValue.version(); @@ -625,7 +632,7 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine { if (versionValue == null) { currentVersion = loadCurrentVersionFromIndex(delete.uid()); } else { - if (versionValue.delete() && (threadPool.estimatedTimeInMillis() - versionValue.time()) > gcDeletesInMillis) { + if (enableGcDeletes && versionValue.delete() && (threadPool.estimatedTimeInMillis() - versionValue.time()) > gcDeletesInMillis) { currentVersion = -1; // deleted, and GC } else { currentVersion = versionValue.version(); @@ -940,7 +947,7 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine { continue; // its a newer value, from after/during we refreshed, don't clear it } if (versionValue.delete()) { - if ((time - versionValue.time()) > gcDeletesInMillis) { + if (enableGcDeletes && (time - versionValue.time()) > gcDeletesInMillis) { versionMap.remove(id); } } else { From 0f9fe35d56f7782916d1d1bc7a302551c2fc5bdf Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 8 Dec 2011 09:47:07 +0100 Subject: [PATCH 002/270] added netbeans ignores --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index fd430544441..b1e82d1d6b6 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,7 @@ target/ */.classpath */.settings */eclipse-build + +## netbeans ignores +nb-configuration.xml +nbactions.xml From bccf0b193e5d2e1a7e1976f08290d35d9e2b4a9b Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 8 Dec 2011 09:48:00 +0100 Subject: [PATCH 003/270] moving debian package to maven --- pom.xml | 125 ++++++++++++++++++++++ src/deb/control/conffiles | 4 + src/deb/control/control | 36 +++++++ src/deb/control/postinst | 39 +++++++ src/deb/control/postrm | 30 ++++++ src/deb/control/prerm | 10 ++ src/deb/default/elasticsearch | 24 +++++ src/deb/init.d/elasticsearch | 192 ++++++++++++++++++++++++++++++++++ 8 files changed, 460 insertions(+) create mode 100644 src/deb/control/conffiles create mode 100644 src/deb/control/control create mode 100755 src/deb/control/postinst create mode 100755 src/deb/control/postrm create mode 100755 src/deb/control/prerm create mode 100644 src/deb/default/elasticsearch create mode 100644 src/deb/init.d/elasticsearch diff --git a/pom.xml b/pom.xml index 4efc6df1e0f..8c6b559e7ed 100644 --- a/pom.xml +++ b/pom.xml @@ -406,6 +406,131 @@ + + maven-dependency-plugin + + + package + + copy-dependencies + + + ${project.build.directory}/lib + + + + + + + jdeb + org.vafer + 0.9-SNAPSHOT + + + package + + jdeb + + + + + + ${project.basedir}/ + *.txt, *.textile + directory + + perm + /usr/share/elasticsearch + root + root + + + + + ${project.basedir}/bin + directory + *.bat + + perm + /usr/share/elasticsearch/bin + 755 + root + root + + + + + ${project.build.directory}/ + ${project.build.finalName}.jar, ${project.basedir}/lib/sigar/* + directory + + perm + /usr/share/elasticsearch/lib + root + root + + + + + ${project.build.directory}/lib + directory + + perm + /usr/share/elasticsearch/lib + root + root + + + + + + ${project.basedir}/src/deb/default/ + directory + + perm + /etc/default + root + root + + + + + ${project.basedir}/src/deb/init.d/ + directory + + perm + /etc/init.d + 755 + root + root + + + + + + ${project.basedir}/config + directory + + perm + /etc/elasticsearch + root + root + + + + + + + + \ No newline at end of file diff --git a/src/deb/control/conffiles b/src/deb/control/conffiles new file mode 100644 index 00000000000..98e0f5a6432 --- /dev/null +++ b/src/deb/control/conffiles @@ -0,0 +1,4 @@ +/etc/init.d/elasticsearch +/etc/default/elasticsearch +/etc/elasticsearch/logging.yml +/etc/elasticsearch/elasticsearch.yml diff --git a/src/deb/control/control b/src/deb/control/control new file mode 100644 index 00000000000..d7026e6ee35 --- /dev/null +++ b/src/deb/control/control @@ -0,0 +1,36 @@ +Package: elasticsearch +Version: [[version]] +Architecture: all +Maintainer: Nicolas Huray +Depends: openjdk-6-jre-headless | sun-java6-jre +Section: web +Priority: optional +Homepage: http://www.elasticsearch.org/ +Description: Open Source, Distributed, RESTful Search Engine + ElasticSearch is a distributed RESTful search engine built for the cloud. + . + Features include: + . + * Distributed and Highly Available Search Engine. + - Each index is fully sharded with a configurable number of shards. + - Each shard can have one or more replicas. + - Read / Search operations performed on either one of the replica shard. + * Multi Tenant with Multi Types. + - Support for more than one index. + - Support for more than one type per index. + - Index level configuration (number of shards, index storage, ...). + * Various set of APIs + - HTTP RESTful API + - Native Java API. + - All APIs perform automatic node operation rerouting. + * Document oriented + - No need for upfront schema definition. + - Schema can be defined per type for customization of the indexing process. + * Reliable, Asynchronous Write Behind for long term persistency. + * (Near) Real Time Search. + * Built on top of Lucene + - Each shard is a fully functional Lucene index + - All the power of Lucene easily exposed through simple configuration / plugins. + * Per operation consistency + - Single document level operations are atomic, consistent, isolated and durable. + * Open Source under Apache 2 License. diff --git a/src/deb/control/postinst b/src/deb/control/postinst new file mode 100755 index 00000000000..85d631b6e23 --- /dev/null +++ b/src/deb/control/postinst @@ -0,0 +1,39 @@ +#!/bin/sh +set -e + +case "$1" in + configure) + [ -f /etc/default/elasticsearch ] && . /etc/default/elasticsearch + [ -z "$ES_USER" ] && ES_USER="elasticsearch" + [ -z "$ES_GROUP" ] && ES_GROUP="elasticsearch" + if ! getent group "$ES_GROUP" > /dev/null 2>&1 ; then + addgroup --system "$ES_GROUP" --quiet + fi + if ! id $ES_USER > /dev/null 2>&1 ; then + adduser --system --home /usr/share/elasticsearch --no-create-home \ + --ingroup "$ES_GROUP" --disabled-password --shell /bin/false \ + "$ES_USER" + fi + + # Set user permissions on /var/log/elasticsearch and /var/lib/elasticsearch + mkdir -p /var/log/elasticsearch /var/lib/elasticsearch + chown -R $ES_USER:$ES_GROUP /var/log/elasticsearch /var/lib/elasticsearch + chmod 755 /var/log/elasticsearch /var/lib/elasticsearch + + # configuration files should not be modifiable by elasticsearch user, as this can be a security issue + chown -Rh root:root /etc/elasticsearch/* + chmod 755 /etc/elasticsearch + chmod 644 /etc/elasticsearch/* + ;; +esac + + +if [ -x "/etc/init.d/elasticsearch" ]; then + update-rc.d elasticsearch defaults 95 10 >/dev/null + if [ -x "`which invoke-rc.d 2>/dev/null`" ]; then + invoke-rc.d elasticsearch start || true + else + /etc/init.d/elasticsearch start || true + fi +fi + diff --git a/src/deb/control/postrm b/src/deb/control/postrm new file mode 100755 index 00000000000..776528a3af4 --- /dev/null +++ b/src/deb/control/postrm @@ -0,0 +1,30 @@ +#!/bin/sh +set -e + +case "$1" in + remove) + # Remove logs and data + rm -rf /var/log/elasticsearch /var/lib/elasticsearch + ;; + + purge) + # Remove service + update-rc.d elasticsearch remove >/dev/null || true + + # Remove logs and data + rm -rf /var/log/elasticsearch /var/lib/elasticsearch + + # Remove user/group + deluser elasticsearch || true + delgroup elasticsearch || true + ;; + + upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + # Nothing to do here + ;; + + *) + echo "$0 called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac diff --git a/src/deb/control/prerm b/src/deb/control/prerm new file mode 100755 index 00000000000..f3603a24df7 --- /dev/null +++ b/src/deb/control/prerm @@ -0,0 +1,10 @@ +#!/bin/sh +set -e + +if [ -x "/etc/init.d/elasticsearch" ]; then + if [ -x "`which invoke-rc.d 2>/dev/null`" ]; then + invoke-rc.d elasticsearch stop || true + else + /etc/init.d/elasticsearch stop || true + fi +fi \ No newline at end of file diff --git a/src/deb/default/elasticsearch b/src/deb/default/elasticsearch new file mode 100644 index 00000000000..f3cecf8af22 --- /dev/null +++ b/src/deb/default/elasticsearch @@ -0,0 +1,24 @@ +# Run ElasticSearch as this user ID and group ID +ES_USER=elasticsearch +ES_GROUP=elasticsearch + +# Minimum Heap memory to run ElasticSearch +ES_MIN_MEM=256m + +# Maximum Heap memory to run ElasticSearch +ES_MAX_MEM=1g + +# ElasticSearch log directory +LOG_DIR=/var/log/elasticsearch + +# ElasticSearch data directory +DATA_DIR=/var/lib/elasticsearch + +# ElasticSearch work directory +WORK_DIR=/tmp/elasticsearch + +# ElasticSearch configuration directory +CONF_DIR=/etc/elasticsearch + +# ElasticSearch configuration file (elasticsearch.yml) +CONF_FILE=/etc/elasticsearch/elasticsearch.yml diff --git a/src/deb/init.d/elasticsearch b/src/deb/init.d/elasticsearch new file mode 100644 index 00000000000..8d250b41847 --- /dev/null +++ b/src/deb/init.d/elasticsearch @@ -0,0 +1,192 @@ +#!/bin/sh +# +# /etc/init.d/elasticsearch -- startup script for Elasticsearch +# +# Written by Miquel van Smoorenburg . +# Modified for Debian GNU/Linux by Ian Murdock . +# Modified for Tomcat by Stefan Gybas . +# Modified for Tomcat6 by Thierry Carrez . +# Additional improvements by Jason Brittain . +# Modified by Nicolas Huray for ElasticSearch . +# +### BEGIN INIT INFO +# Provides: elasticsearch +# Required-Start: $all +# Required-Stop: $all +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts elasticsearch +# Description: Starts elasticsearch using start-stop-daemon +### END INIT INFO + +set -e + +PATH=/bin:/usr/bin:/sbin:/usr/sbin +NAME=elasticsearch +DESC="ElasticSearch Server" +DEFAULT=/etc/default/$NAME + +if [ `id -u` -ne 0 ]; then + echo "You need root privileges to run this script" + exit 1 +fi + + +. /lib/lsb/init-functions + +if [ -r /etc/default/rcS ]; then + . /etc/default/rcS +fi + + +# The following variables can be overwritten in $DEFAULT + +# Run ElasticSearch as this user ID and group ID +ES_USER=elasticsearch +ES_GROUP=elasticsearch + +# The first existing directory is used for JAVA_HOME (if JAVA_HOME is not defined in $DEFAULT) +JDK_DIRS="/usr/lib/jvm/java-7-openjdk /usr/lib/jvm/java-7-oracle /usr/lib/jvm/java-6-openjdk /usr/lib/jvm/java-6-sun" + +# Look for the right JVM to use +for jdir in $JDK_DIRS; do + if [ -r "$jdir/bin/java" -a -z "${JAVA_HOME}" ]; then + JAVA_HOME="$jdir" + fi +done +export JAVA_HOME + +# Directory where the ElasticSearch binary distribution resides +ES_HOME=/usr/share/$NAME + +# Minimum Heap memory to run ElasticSearch +ES_MIN_MEM=256m + +# Maximum Heap memory to run ElasticSearch +ES_MAX_MEM=1g + +# ElasticSearch log directory +LOG_DIR=/var/log/$NAME + +# ElasticSearch data directory +DATA_DIR=/var/lib/$NAME + +# ElasticSearch work directory +WORK_DIR=/tmp/$NAME + +# ElasticSearch configuration directory +CONF_DIR=/etc/$NAME + +# ElasticSearch configuration file (elasticsearch.yml) +CONF_FILE=$CONF_DIR/elasticsearch.yml + +# End of variables that can be overwritten in $DEFAULT + +# overwrite settings from default file +if [ -f "$DEFAULT" ]; then + . "$DEFAULT" +fi + +# Define other required variables +PID_FILE=/var/run/$NAME.pid +DAEMON=$ES_HOME/bin/elasticsearch +DAEMON_OPTS="-p $PID_FILE -Des.config=$CONF_FILE -Des.path.home=$ES_HOME -Des.path.logs=$LOG_DIR -Des.path.data=$DATA_DIR -Des.path.work=$WORK_DIR -Des.path.conf=$CONF_DIR" + +export ES_MIN_MEM ES_MAX_MEM + +# Check DAEMON exists +test -x $DAEMON || exit 0 + +case "$1" in + start) + if [ -z "$JAVA_HOME" ]; then + log_failure_msg "no JDK found - please set JAVA_HOME" + exit 1 + fi + + log_daemon_msg "Starting $DESC" + + if start-stop-daemon --test --start --pidfile "$PID_FILE" \ + --user "$ES_USER" --exec "$JAVA_HOME/bin/java" \ + >/dev/null; then + + # Prepare environment + mkdir -p "$LOG_DIR" "$DATA_DIR" "$WORK_DIR" && chown "$ES_USER":"$ES_GROUP" "$LOG_DIR" "$DATA_DIR" "$WORK_DIR" + touch "$PID_FILE" && chown "$ES_USER":"$ES_GROUP" "$PID_FILE" + ulimit -n 65535 + + # Start Daemon + start-stop-daemon --start -b --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec /bin/bash -- -c "$DAEMON $DAEMON_OPTS" + + sleep 1 + if start-stop-daemon --test --start --pidfile "$PID_FILE" \ + --user "$ES_USER" --exec "$JAVA_HOME/bin/java" \ + >/dev/null; then + if [ -f "$PID_FILE" ]; then + rm -f "$PID_FILE" + fi + log_end_msg 1 + else + log_end_msg 0 + fi + + else + log_progress_msg "(already running)" + log_end_msg 0 + fi + ;; + stop) + log_daemon_msg "Stopping $DESC" + + set +e + if [ -f "$PID_FILE" ]; then + start-stop-daemon --stop --pidfile "$PID_FILE" \ + --user "$ES_USER" \ + --retry=TERM/20/KILL/5 >/dev/null + if [ $? -eq 1 ]; then + log_progress_msg "$DESC is not running but pid file exists, cleaning up" + elif [ $? -eq 3 ]; then + PID="`cat $PID_FILE`" + log_failure_msg "Failed to stop $DESC (pid $PID)" + exit 1 + fi + rm -f "$PID_FILE" + else + log_progress_msg "(not running)" + fi + log_end_msg 0 + set -e + ;; + status) + set +e + start-stop-daemon --test --start --pidfile "$PID_FILE" \ + --user "$ES_USER" --exec "$JAVA_HOME/bin/java" \ + >/dev/null 2>&1 + if [ "$?" = "0" ]; then + + if [ -f "$PID_FILE" ]; then + log_success_msg "$DESC is not running, but pid file exists." + exit 1 + else + log_success_msg "$DESC is not running." + exit 3 + fi + else + log_success_msg "$DESC is running with pid `cat $PID_FILE`" + fi + set -e + ;; + restart|force-reload) + if [ -f "$PID_FILE" ]; then + $0 stop + sleep 1 + fi + $0 start + ;; + *) + log_success_msg "Usage: $0 {start|stop|restart|force-reload|status}" + exit 1 + ;; +esac + +exit 0 From 6d0b94b81be80878f4af0441651980ce91d33b85 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 8 Dec 2011 12:05:05 +0100 Subject: [PATCH 004/270] improved deb packaging and bound assembly:single to package --- pom.xml | 56 ++++++++++++++++++++++++++------------------------------ 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/pom.xml b/pom.xml index 8c6b559e7ed..f27cc76be38 100644 --- a/pom.xml +++ b/pom.xml @@ -399,33 +399,44 @@ maven-assembly-plugin - ${project.build.directory}/releases/ + ${project.build.directory}/releases/ ${basedir}/src/main/assemblies/targz-bin.xml ${basedir}/src/main/assemblies/zip-bin.xml - + - + + + package + + single + + + + maven-dependency-plugin - package + prepare-package copy-dependencies ${project.build.directory}/lib - + - + jdeb org.vafer - 0.9-SNAPSHOT + 0.8 + + ${project.build.directory}/releases/${artifactId}-${version}.deb + package @@ -433,9 +444,7 @@ jdeb - - + ${project.basedir}/ *.txt, *.textile @@ -447,7 +456,6 @@ root - ${project.basedir}/bin directory @@ -459,8 +467,7 @@ root root - - + ${project.build.directory}/ ${project.build.finalName}.jar, ${project.basedir}/lib/sigar/* @@ -472,9 +479,9 @@ root - - + ${project.build.directory}/lib + lucene*, log4j*, jline*, jna* directory perm @@ -482,11 +489,7 @@ root root - - - + ${project.basedir}/src/deb/default/ directory @@ -496,10 +499,7 @@ root root - - + ${project.basedir}/src/deb/init.d/ directory @@ -510,11 +510,7 @@ root root - - - + ${project.basedir}/config directory @@ -530,7 +526,7 @@ - + \ No newline at end of file From 16c2c6b2c37eca480239f0146ebaec0dc5fa0a83 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 8 Dec 2011 13:40:09 +0200 Subject: [PATCH 005/270] fix deb to properly copy over sigar --- pom.xml | 69 +++++++++++++++++------------- src/main/assemblies/common-bin.xml | 4 ++ 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/pom.xml b/pom.xml index f27cc76be38..b671570df2b 100644 --- a/pom.xml +++ b/pom.xml @@ -399,21 +399,21 @@ maven-assembly-plugin - ${project.build.directory}/releases/ + ${project.build.directory}/releases/ ${basedir}/src/main/assemblies/targz-bin.xml ${basedir}/src/main/assemblies/zip-bin.xml - + - + package single - - + + maven-dependency-plugin @@ -424,19 +424,19 @@ ${project.build.directory}/lib - + - + jdeb org.vafer 0.8 - ${project.build.directory}/releases/${artifactId}-${version}.deb - + ${project.build.directory}/releases/${project.artifactId}-${project.version}.deb + package @@ -444,33 +444,33 @@ jdeb - - + + ${project.basedir}/ *.txt, *.textile directory perm - /usr/share/elasticsearch + /usr/share/elasticsearch root root - - + + ${project.basedir}/bin directory *.bat perm - /usr/share/elasticsearch/bin + /usr/share/elasticsearch/bin 755 root root - - - + + + ${project.build.directory}/ - ${project.build.finalName}.jar, ${project.basedir}/lib/sigar/* + ${project.build.finalName}.jar directory perm @@ -478,9 +478,20 @@ root root - + - ${project.build.directory}/lib + ${project.basedir}/lib/sigar/ + sigar-*.jar, libsigar-*-linux.* + directory + + perm + /usr/share/elasticsearch/lib/sigar + root + root + + + + ${project.build.directory}/lib lucene*, log4j*, jline*, jna* directory @@ -490,17 +501,17 @@ root - + ${project.basedir}/src/deb/default/ directory perm - /etc/default + /etc/default root root - + ${project.basedir}/src/deb/init.d/ directory @@ -511,22 +522,22 @@ root - + ${project.basedir}/config directory perm - /etc/elasticsearch + /etc/elasticsearch root root - - + + - + \ No newline at end of file diff --git a/src/main/assemblies/common-bin.xml b/src/main/assemblies/common-bin.xml index 130fef6c810..c6df88a26ac 100644 --- a/src/main/assemblies/common-bin.xml +++ b/src/main/assemblies/common-bin.xml @@ -28,6 +28,10 @@ + + README.textile + / + LICENSE.txt / From 074c70dc4f73f222a1998ab84d8abb202984bdb3 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 8 Dec 2011 13:40:39 +0200 Subject: [PATCH 006/270] building dist should be using package --- README.textile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.textile b/README.textile index 18d94055300..54d030c1201 100644 --- a/README.textile +++ b/README.textile @@ -198,7 +198,7 @@ h3. Building from Source ElasticSearch uses "Maven":http://maven.apache.org for its build system. -In order to create a distribution, simply run the @mvn assembly:assembly -DskipTests@ command in the cloned directory. +In order to create a distribution, simply run the @mvn package -DskipTests@ command in the cloned directory. The distribution will be created under @target/releases@. From b7d3f8fc99b50f35461d26c9ece7682256a6eb2a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 8 Dec 2011 14:10:25 +0200 Subject: [PATCH 007/270] use the smaller value possible --- .../replication/TransportShardReplicationOperationAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java b/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java index 3286cf106b4..3e7f7dce0e6 100644 --- a/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java @@ -533,7 +533,7 @@ public abstract class TransportShardReplicationOperationAction Date: Thu, 8 Dec 2011 14:14:04 +0200 Subject: [PATCH 008/270] separate writeAllowed check to actual operation compared to optimize/flush/refresh --- .../index/shard/service/InternalIndexShard.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java index 5590cfec71c..34756755fd8 100644 --- a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java +++ b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java @@ -410,7 +410,7 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I @Override public void refresh(Engine.Refresh refresh) throws ElasticSearchException { - writeAllowed(); + verifyStarted(); if (logger.isTraceEnabled()) { logger.trace("refresh with {}", refresh); } @@ -475,7 +475,7 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I @Override public void flush(Engine.Flush flush) throws ElasticSearchException { - writeAllowed(); + verifyStarted(); if (logger.isTraceEnabled()) { logger.trace("flush with {}", flush); } @@ -486,7 +486,7 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I @Override public void optimize(Engine.Optimize optimize) throws ElasticSearchException { - writeAllowed(); + verifyStarted(); if (logger.isTraceEnabled()) { logger.trace("optimize with {}", optimize); } @@ -505,7 +505,7 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I @Override public void recover(Engine.RecoveryHandler recoveryHandler) throws EngineException { - writeAllowed(); + verifyStarted(); engine.recover(recoveryHandler); } @@ -648,6 +648,13 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I } } + public void verifyStarted() throws IllegalIndexShardStateException { + IndexShardState state = this.state; // one time volatile read + if (state != IndexShardState.STARTED) { + throw new IndexShardNotStartedException(shardId, state); + } + } + private void startScheduledTasksIfNeeded() { if (refreshInterval.millis() > 0) { refreshScheduledFuture = threadPool.schedule(refreshInterval, ThreadPool.Names.SAME, new EngineRefresher()); From 6585b09de6ca339ea355d1d9ed5f56e7b2d0a9e4 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 8 Dec 2011 16:28:04 +0200 Subject: [PATCH 009/270] add link to download in readme --- README.textile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.textile b/README.textile index 54d030c1201..808320fc70e 100644 --- a/README.textile +++ b/README.textile @@ -36,7 +36,7 @@ First of all, DON'T PANIC. It will take 5 minutes to get the gist of what Elasti h3. Installation -* Download and unzip the ElasticSearch installation. +* "Download":http://www.elasticsearch.org/download and unzip the ElasticSearch official distribution. * Run @bin/elasticsearch -f@ on unix, or @bin/elasticsearch.bat@ on windows. * Run @curl -X GET http://localhost:9200/@. * Start more servers ... From 1fd5a48409635722860f6d923406d745bfb2dd32 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 8 Dec 2011 16:28:57 +0200 Subject: [PATCH 010/270] wait for yellow status before searching --- .../test/integration/search/query/SimpleQueryTests.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java b/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java index 8367edaeb88..b77a65b94be 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java @@ -323,6 +323,9 @@ public class SimpleQueryTests extends AbstractNodesTests { client.admin().indices().prepareDelete().execute().actionGet(); client.prepareIndex("test", "type1", "1").setSource("field1", "value1").execute().actionGet(); + + client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); + client.admin().indices().prepareRefresh().execute().actionGet(); SearchResponse searchResponse = client.prepareSearch("test").setQuery(constantScoreQuery(termsFilter("field1", "value1").cacheKey("test1"))).execute().actionGet(); From 1cd3af9de0938e6f37bb97b87daedd927bed388b Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 8 Dec 2011 17:40:43 +0200 Subject: [PATCH 011/270] better exception message --- src/main/java/org/elasticsearch/env/NodeEnvironment.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/src/main/java/org/elasticsearch/env/NodeEnvironment.java index cf29afa36c9..37ae247c372 100644 --- a/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.shard.ShardId; import java.io.File; import java.io.IOException; +import java.util.Arrays; /** * @@ -116,7 +117,7 @@ public class NodeEnvironment extends AbstractComponent { } } if (locks[0] == null) { - throw new IOException("Failed to obtain node lock", lastException); + throw new IOException("Failed to obtain node lock on " + Arrays.toString(environment.dataWithClusterFiles()), lastException); } this.localNodeId = localNodeId; From e56086cf7ba95de7a1779d3b5f0e22044f0ec18f Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 8 Dec 2011 17:41:27 +0200 Subject: [PATCH 012/270] allow to perform operations on a recovering shard using versioning to do conflict detection --- .../index/engine/robin/RobinEngine.java | 418 +++++++++--------- .../shard/service/InternalIndexShard.java | 21 +- .../index/translog/TranslogService.java | 4 +- 3 files changed, 217 insertions(+), 226 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java b/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java index 2a285d33c73..7c818db6a92 100644 --- a/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java +++ b/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java @@ -374,97 +374,94 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine { private void innerCreate(Create create, IndexWriter writer) throws IOException { synchronized (dirtyLock(create.uid())) { UidField uidField = create.uidField(); - if (create.origin() == Operation.Origin.RECOVERY) { - uidField.version(create.version()); - // we use update doc and not addDoc since we might get duplicates when using transient translog - if (create.docs().size() > 1) { - writer.updateDocuments(create.uid(), create.docs(), create.analyzer()); - } else { - writer.updateDocument(create.uid(), create.docs().get(0), create.analyzer()); - } - Translog.Location translogLocation = translog.add(new Translog.Create(create)); - // on recovery, we get the actual version we want to use - if (create.version() != 0) { - versionMap.put(create.uid().text(), new VersionValue(create.version(), false, threadPool.estimatedTimeInMillis(), translogLocation)); - } + final long currentVersion; + VersionValue versionValue = versionMap.get(create.uid().text()); + if (versionValue == null) { + currentVersion = loadCurrentVersionFromIndex(create.uid()); } else { - long currentVersion; - VersionValue versionValue = versionMap.get(create.uid().text()); - if (versionValue == null) { - currentVersion = loadCurrentVersionFromIndex(create.uid()); + if (enableGcDeletes && versionValue.delete() && (threadPool.estimatedTimeInMillis() - versionValue.time()) > gcDeletesInMillis) { + currentVersion = -1; // deleted, and GC } else { - if (enableGcDeletes && versionValue.delete() && (threadPool.estimatedTimeInMillis() - versionValue.time()) > gcDeletesInMillis) { - currentVersion = -1; // deleted, and GC - } else { - currentVersion = versionValue.version(); - } + currentVersion = versionValue.version(); } + } - // same logic as index - long updatedVersion; - if (create.origin() == Operation.Origin.PRIMARY) { - if (create.versionType() == VersionType.INTERNAL) { // internal version type - long expectedVersion = create.version(); - if (expectedVersion != 0 && currentVersion != -2) { // -2 means we don't have a version, so ignore... - // an explicit version is provided, see if there is a conflict - // if the current version is -1, means we did not find anything, and - // a version is provided, so we do expect to find a doc under that version - // this is important, since we don't allow to preset a version in order to handle deletes - if (currentVersion == -1) { - throw new VersionConflictEngineException(shardId, create.type(), create.id(), -1, expectedVersion); - } else if (expectedVersion != currentVersion) { - throw new VersionConflictEngineException(shardId, create.type(), create.id(), currentVersion, expectedVersion); - } - } - updatedVersion = currentVersion < 0 ? 1 : currentVersion + 1; - } else { // external version type - // an external version is provided, just check, if a local version exists, that its higher than it - // the actual version checking is one in an external system, and we just want to not index older versions - if (currentVersion >= 0) { // we can check!, its there - if (currentVersion >= create.version()) { - throw new VersionConflictEngineException(shardId, create.type(), create.id(), currentVersion, create.version()); - } - } - updatedVersion = create.version(); - } - } else { // if (index.origin() == Operation.Origin.REPLICA) { + // same logic as index + long updatedVersion; + if (create.origin() == Operation.Origin.PRIMARY) { + if (create.versionType() == VersionType.INTERNAL) { // internal version type long expectedVersion = create.version(); - if (currentVersion != -2) { // -2 means we don't have a version, so ignore... - // if it does not exists, and its considered the first index operation (replicas are 1 of) - // then nothing to do - if (!(currentVersion == -1 && create.version() == 1)) { - // with replicas, we only check for previous version, we allow to set a future version - if (expectedVersion <= currentVersion) { - throw new VersionConflictEngineException(shardId, create.type(), create.id(), currentVersion, expectedVersion); - } + if (expectedVersion != 0 && currentVersion != -2) { // -2 means we don't have a version, so ignore... + // an explicit version is provided, see if there is a conflict + // if the current version is -1, means we did not find anything, and + // a version is provided, so we do expect to find a doc under that version + // this is important, since we don't allow to preset a version in order to handle deletes + if (currentVersion == -1) { + throw new VersionConflictEngineException(shardId, create.type(), create.id(), -1, expectedVersion); + } else if (expectedVersion != currentVersion) { + throw new VersionConflictEngineException(shardId, create.type(), create.id(), currentVersion, expectedVersion); + } + } + updatedVersion = currentVersion < 0 ? 1 : currentVersion + 1; + } else { // external version type + // an external version is provided, just check, if a local version exists, that its higher than it + // the actual version checking is one in an external system, and we just want to not index older versions + if (currentVersion >= 0) { // we can check!, its there + if (currentVersion >= create.version()) { + throw new VersionConflictEngineException(shardId, create.type(), create.id(), currentVersion, create.version()); } } - // replicas already hold the "future" version updatedVersion = create.version(); } + } else { // if (index.origin() == Operation.Origin.REPLICA || index.origin() == Operation.Origin.RECOVERY) { + long expectedVersion = create.version(); + if (currentVersion != -2) { // -2 means we don't have a version, so ignore... + // if it does not exists, and its considered the first index operation (replicas/recovery are 1 of) + // then nothing to check + if (!(currentVersion == -1 && create.version() == 1)) { + // with replicas/recovery, we only check for previous version, we allow to set a future version + if (expectedVersion <= currentVersion) { + if (create.origin() == Operation.Origin.RECOVERY) { + return; + } else { + throw new VersionConflictEngineException(shardId, create.type(), create.id(), currentVersion, expectedVersion); + } + } + } + } + // replicas already hold the "future" version + updatedVersion = create.version(); + } - // if the doc does not exists or it exists but not delete - if (versionValue != null) { - if (!versionValue.delete()) { + // if the doc does not exists or it exists but not delete + if (versionValue != null) { + if (!versionValue.delete()) { + if (create.origin() == Operation.Origin.RECOVERY) { + return; + } else { throw new DocumentAlreadyExistsEngineException(shardId, create.type(), create.id()); } - } else if (currentVersion != -1) { - // its not deleted, its already there + } + } else if (currentVersion != -1) { + // its not deleted, its already there + if (create.origin() == Operation.Origin.RECOVERY) { + return; + } else { throw new DocumentAlreadyExistsEngineException(shardId, create.type(), create.id()); } - - uidField.version(updatedVersion); - create.version(updatedVersion); - - if (create.docs().size() > 1) { - writer.addDocuments(create.docs(), create.analyzer()); - } else { - writer.addDocument(create.docs().get(0), create.analyzer()); - } - Translog.Location translogLocation = translog.add(new Translog.Create(create)); - - versionMap.put(create.uid().text(), new VersionValue(updatedVersion, false, threadPool.estimatedTimeInMillis(), translogLocation)); } + + uidField.version(updatedVersion); + create.version(updatedVersion); + + if (create.docs().size() > 1) { + writer.addDocuments(create.docs(), create.analyzer()); + } else { + writer.addDocument(create.docs().get(0), create.analyzer()); + } + Translog.Location translogLocation = translog.add(new Translog.Create(create)); + + versionMap.put(create.uid().text(), new VersionValue(updatedVersion, false, threadPool.estimatedTimeInMillis(), translogLocation)); } } @@ -499,94 +496,84 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine { private void innerIndex(Index index, IndexWriter writer) throws IOException { synchronized (dirtyLock(index.uid())) { UidField uidField = index.uidField(); - if (index.origin() == Operation.Origin.RECOVERY) { - uidField.version(index.version()); + final long currentVersion; + VersionValue versionValue = versionMap.get(index.uid().text()); + if (versionValue == null) { + currentVersion = loadCurrentVersionFromIndex(index.uid()); + } else { + if (enableGcDeletes && versionValue.delete() && (threadPool.estimatedTimeInMillis() - versionValue.time()) > gcDeletesInMillis) { + currentVersion = -1; // deleted, and GC + } else { + currentVersion = versionValue.version(); + } + } + + long updatedVersion; + if (index.origin() == Operation.Origin.PRIMARY) { + if (index.versionType() == VersionType.INTERNAL) { // internal version type + long expectedVersion = index.version(); + if (expectedVersion != 0 && currentVersion != -2) { // -2 means we don't have a version, so ignore... + // an explicit version is provided, see if there is a conflict + // if the current version is -1, means we did not find anything, and + // a version is provided, so we do expect to find a doc under that version + // this is important, since we don't allow to preset a version in order to handle deletes + if (currentVersion == -1) { + throw new VersionConflictEngineException(shardId, index.type(), index.id(), -1, expectedVersion); + } else if (expectedVersion != currentVersion) { + throw new VersionConflictEngineException(shardId, index.type(), index.id(), currentVersion, expectedVersion); + } + } + updatedVersion = currentVersion < 0 ? 1 : currentVersion + 1; + } else { // external version type + // an external version is provided, just check, if a local version exists, that its higher than it + // the actual version checking is one in an external system, and we just want to not index older versions + if (currentVersion >= 0) { // we can check!, its there + if (currentVersion >= index.version()) { + throw new VersionConflictEngineException(shardId, index.type(), index.id(), currentVersion, index.version()); + } + } + updatedVersion = index.version(); + } + } else { // if (index.origin() == Operation.Origin.REPLICA || index.origin() == Operation.Origin.RECOVERY) { + long expectedVersion = index.version(); + if (currentVersion != -2) { // -2 means we don't have a version, so ignore... + // if it does not exists, and its considered the first index operation (replicas/recovery are 1 of) + // then nothing to check + if (!(currentVersion == -1 && index.version() == 1)) { + // with replicas/recovery, we only check for previous version, we allow to set a future version + if (expectedVersion <= currentVersion) { + if (index.origin() == Operation.Origin.RECOVERY) { + return; + } else { + throw new VersionConflictEngineException(shardId, index.type(), index.id(), currentVersion, expectedVersion); + } + } + } + } + // replicas already hold the "future" version + updatedVersion = index.version(); + } + + uidField.version(updatedVersion); + index.version(updatedVersion); + + if (currentVersion == -1) { + // document does not exists, we can optimize for create + if (index.docs().size() > 1) { + writer.addDocuments(index.docs(), index.analyzer()); + } else { + writer.addDocument(index.docs().get(0), index.analyzer()); + } + } else { if (index.docs().size() > 1) { writer.updateDocuments(index.uid(), index.docs(), index.analyzer()); } else { writer.updateDocument(index.uid(), index.docs().get(0), index.analyzer()); } - Translog.Location translogLocation = translog.add(new Translog.Index(index)); - // on recovery, we get the actual version we want to use - if (index.version() != 0) { - versionMap.put(index.uid().text(), new VersionValue(index.version(), false, threadPool.estimatedTimeInMillis(), translogLocation)); - } - } else { - long currentVersion; - VersionValue versionValue = versionMap.get(index.uid().text()); - if (versionValue == null) { - currentVersion = loadCurrentVersionFromIndex(index.uid()); - } else { - if (enableGcDeletes && versionValue.delete() && (threadPool.estimatedTimeInMillis() - versionValue.time()) > gcDeletesInMillis) { - currentVersion = -1; // deleted, and GC - } else { - currentVersion = versionValue.version(); - } - } - - long updatedVersion; - if (index.origin() == Operation.Origin.PRIMARY) { - if (index.versionType() == VersionType.INTERNAL) { // internal version type - long expectedVersion = index.version(); - if (expectedVersion != 0 && currentVersion != -2) { // -2 means we don't have a version, so ignore... - // an explicit version is provided, see if there is a conflict - // if the current version is -1, means we did not find anything, and - // a version is provided, so we do expect to find a doc under that version - // this is important, since we don't allow to preset a version in order to handle deletes - if (currentVersion == -1) { - throw new VersionConflictEngineException(shardId, index.type(), index.id(), -1, expectedVersion); - } else if (expectedVersion != currentVersion) { - throw new VersionConflictEngineException(shardId, index.type(), index.id(), currentVersion, expectedVersion); - } - } - updatedVersion = currentVersion < 0 ? 1 : currentVersion + 1; - } else { // external version type - // an external version is provided, just check, if a local version exists, that its higher than it - // the actual version checking is one in an external system, and we just want to not index older versions - if (currentVersion >= 0) { // we can check!, its there - if (currentVersion >= index.version()) { - throw new VersionConflictEngineException(shardId, index.type(), index.id(), currentVersion, index.version()); - } - } - updatedVersion = index.version(); - } - } else { // if (index.origin() == Operation.Origin.REPLICA) { - long expectedVersion = index.version(); - if (currentVersion != -2) { // -2 means we don't have a version, so ignore... - // if it does not exists, and its considered the first index operation (replicas are 1 of) - // then nothing to do - if (!(currentVersion == -1 && index.version() == 1)) { - // with replicas, we only check for previous version, we allow to set a future version - if (expectedVersion <= currentVersion) { - throw new VersionConflictEngineException(shardId, index.type(), index.id(), currentVersion, expectedVersion); - } - } - } - // replicas already hold the "future" version - updatedVersion = index.version(); - } - - uidField.version(updatedVersion); - index.version(updatedVersion); - - if (currentVersion == -1) { - // document does not exists, we can optimize for create - if (index.docs().size() > 1) { - writer.addDocuments(index.docs(), index.analyzer()); - } else { - writer.addDocument(index.docs().get(0), index.analyzer()); - } - } else { - if (index.docs().size() > 1) { - writer.updateDocuments(index.uid(), index.docs(), index.analyzer()); - } else { - writer.updateDocument(index.uid(), index.docs().get(0), index.analyzer()); - } - } - Translog.Location translogLocation = translog.add(new Translog.Index(index)); - - versionMap.put(index.uid().text(), new VersionValue(updatedVersion, false, threadPool.estimatedTimeInMillis(), translogLocation)); } + Translog.Location translogLocation = translog.add(new Translog.Index(index)); + + versionMap.put(index.uid().text(), new VersionValue(updatedVersion, false, threadPool.estimatedTimeInMillis(), translogLocation)); } } @@ -619,80 +606,75 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine { private void innerDelete(Delete delete, IndexWriter writer) throws IOException { synchronized (dirtyLock(delete.uid())) { - if (delete.origin() == Operation.Origin.RECOVERY) { - writer.deleteDocuments(delete.uid()); - Translog.Location translogLocation = translog.add(new Translog.Delete(delete)); - // update the version with the exact version from recovery, assuming we have it - if (delete.version() != 0) { - versionMap.put(delete.uid().text(), new VersionValue(delete.version(), true, threadPool.estimatedTimeInMillis(), translogLocation)); - } + final long currentVersion; + VersionValue versionValue = versionMap.get(delete.uid().text()); + if (versionValue == null) { + currentVersion = loadCurrentVersionFromIndex(delete.uid()); } else { - long currentVersion; - VersionValue versionValue = versionMap.get(delete.uid().text()); - if (versionValue == null) { - currentVersion = loadCurrentVersionFromIndex(delete.uid()); + if (enableGcDeletes && versionValue.delete() && (threadPool.estimatedTimeInMillis() - versionValue.time()) > gcDeletesInMillis) { + currentVersion = -1; // deleted, and GC } else { - if (enableGcDeletes && versionValue.delete() && (threadPool.estimatedTimeInMillis() - versionValue.time()) > gcDeletesInMillis) { - currentVersion = -1; // deleted, and GC - } else { - currentVersion = versionValue.version(); - } + currentVersion = versionValue.version(); } + } - long updatedVersion; - if (delete.origin() == Operation.Origin.PRIMARY) { - if (delete.versionType() == VersionType.INTERNAL) { // internal version type - if (delete.version() != 0 && currentVersion != -2) { // -2 means we don't have a version, so ignore... - // an explicit version is provided, see if there is a conflict - // if the current version is -1, means we did not find anything, and - // a version is provided, so we do expect to find a doc under that version - if (currentVersion == -1) { - throw new VersionConflictEngineException(shardId, delete.type(), delete.id(), -1, delete.version()); - } else if (delete.version() != currentVersion) { - throw new VersionConflictEngineException(shardId, delete.type(), delete.id(), currentVersion, delete.version()); - } - } - updatedVersion = currentVersion < 0 ? 1 : currentVersion + 1; - } else { // External + long updatedVersion; + if (delete.origin() == Operation.Origin.PRIMARY) { + if (delete.versionType() == VersionType.INTERNAL) { // internal version type + if (delete.version() != 0 && currentVersion != -2) { // -2 means we don't have a version, so ignore... + // an explicit version is provided, see if there is a conflict + // if the current version is -1, means we did not find anything, and + // a version is provided, so we do expect to find a doc under that version if (currentVersion == -1) { - // its an external version, that's fine, we allow it to be set - //throw new VersionConflictEngineException(shardId, delete.type(), delete.id(), -1, delete.version()); - } else if (currentVersion >= delete.version()) { + throw new VersionConflictEngineException(shardId, delete.type(), delete.id(), -1, delete.version()); + } else if (delete.version() != currentVersion) { throw new VersionConflictEngineException(shardId, delete.type(), delete.id(), currentVersion, delete.version()); } - updatedVersion = delete.version(); } - } else { // if (delete.origin() == Operation.Origin.REPLICA) { - // on replica, the version is the future value expected (returned from the operation on the primary) - if (currentVersion != -2) { // -2 means we don't have a version in the index, ignore - // only check if we have a version for it, otherwise, ignore (see later) - if (currentVersion != -1) { - // with replicas, we only check for previous version, we allow to set a future version - if (delete.version() <= currentVersion) { + updatedVersion = currentVersion < 0 ? 1 : currentVersion + 1; + } else { // External + if (currentVersion == -1) { + // its an external version, that's fine, we allow it to be set + //throw new VersionConflictEngineException(shardId, delete.type(), delete.id(), -1, delete.version()); + } else if (currentVersion >= delete.version()) { + throw new VersionConflictEngineException(shardId, delete.type(), delete.id(), currentVersion, delete.version()); + } + updatedVersion = delete.version(); + } + } else { // if (index.origin() == Operation.Origin.REPLICA || index.origin() == Operation.Origin.RECOVERY) { + // on replica, the version is the future value expected (returned from the operation on the primary) + if (currentVersion != -2) { // -2 means we don't have a version in the index, ignore + // only check if we have a version for it, otherwise, ignore (see later) + if (currentVersion != -1) { + // with replicas, we only check for previous version, we allow to set a future version + if (delete.version() <= currentVersion) { + if (delete.origin() == Operation.Origin.RECOVERY) { + return; + } else { throw new VersionConflictEngineException(shardId, delete.type(), delete.id(), currentVersion - 1, delete.version()); } } } - // replicas already hold the "future" version - updatedVersion = delete.version(); } + // replicas already hold the "future" version + updatedVersion = delete.version(); + } - if (currentVersion == -1) { - // doc does not exists and no prior deletes - delete.version(updatedVersion).notFound(true); - Translog.Location translogLocation = translog.add(new Translog.Delete(delete)); - versionMap.put(delete.uid().text(), new VersionValue(updatedVersion, true, threadPool.estimatedTimeInMillis(), translogLocation)); - } else if (versionValue != null && versionValue.delete()) { - // a "delete on delete", in this case, we still increment the version, log it, and return that version - delete.version(updatedVersion).notFound(true); - Translog.Location translogLocation = translog.add(new Translog.Delete(delete)); - versionMap.put(delete.uid().text(), new VersionValue(updatedVersion, true, threadPool.estimatedTimeInMillis(), translogLocation)); - } else { - delete.version(updatedVersion); - writer.deleteDocuments(delete.uid()); - Translog.Location translogLocation = translog.add(new Translog.Delete(delete)); - versionMap.put(delete.uid().text(), new VersionValue(updatedVersion, true, threadPool.estimatedTimeInMillis(), translogLocation)); - } + if (currentVersion == -1) { + // doc does not exists and no prior deletes + delete.version(updatedVersion).notFound(true); + Translog.Location translogLocation = translog.add(new Translog.Delete(delete)); + versionMap.put(delete.uid().text(), new VersionValue(updatedVersion, true, threadPool.estimatedTimeInMillis(), translogLocation)); + } else if (versionValue != null && versionValue.delete()) { + // a "delete on delete", in this case, we still increment the version, log it, and return that version + delete.version(updatedVersion).notFound(true); + Translog.Location translogLocation = translog.add(new Translog.Delete(delete)); + versionMap.put(delete.uid().text(), new VersionValue(updatedVersion, true, threadPool.estimatedTimeInMillis(), translogLocation)); + } else { + delete.version(updatedVersion); + writer.deleteDocuments(delete.uid()); + Translog.Location translogLocation = translog.add(new Translog.Delete(delete)); + versionMap.put(delete.uid().text(), new VersionValue(updatedVersion, true, threadPool.estimatedTimeInMillis(), translogLocation)); } } } diff --git a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java index 34756755fd8..7a82e0d0673 100644 --- a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java +++ b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java @@ -475,7 +475,10 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I @Override public void flush(Engine.Flush flush) throws ElasticSearchException { - verifyStarted(); + // we allows flush while recovering, since we allow for operations to happen + // while recovering, and we want to keep the translog at bay (up to deletes, which + // we don't gc). + verifyStartedOrRecovering(); if (logger.isTraceEnabled()) { logger.trace("flush with {}", flush); } @@ -544,6 +547,9 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I if (checkIndexOnStartup) { checkIndex(true); } + // we disable deletes since we allow for operations to be executed against the shard while recovering + // but we need to make sure we don't loose deletes until we are done recovering + engine.enableGcDeletes(false); engine.start(); } @@ -572,6 +578,7 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I } startScheduledTasksIfNeeded(); indicesLifecycle.afterIndexShardStarted(this); + engine.enableGcDeletes(true); } public void performRecoveryOperation(Translog.Operation operation) throws ElasticSearchException { @@ -641,14 +648,18 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I } } - public void writeAllowed() throws IllegalIndexShardStateException { + private void writeAllowed() throws IllegalIndexShardStateException { + verifyStartedOrRecovering(); + } + + private void verifyStartedOrRecovering() throws IllegalIndexShardStateException { IndexShardState state = this.state; // one time volatile read - if (state != IndexShardState.STARTED) { - throw new IndexShardNotStartedException(shardId, state); + if (state != IndexShardState.STARTED && state != IndexShardState.RECOVERING) { + throw new IllegalIndexShardStateException(shardId, state, "write operation only allowed when started/recovering"); } } - public void verifyStarted() throws IllegalIndexShardStateException { + private void verifyStarted() throws IllegalIndexShardStateException { IndexShardState state = this.state; // one time volatile read if (state != IndexShardState.STARTED) { throw new IndexShardNotStartedException(shardId, state); diff --git a/src/main/java/org/elasticsearch/index/translog/TranslogService.java b/src/main/java/org/elasticsearch/index/translog/TranslogService.java index 48365f4f868..bc1cad9528b 100644 --- a/src/main/java/org/elasticsearch/index/translog/TranslogService.java +++ b/src/main/java/org/elasticsearch/index/translog/TranslogService.java @@ -170,9 +170,7 @@ public class TranslogService extends AbstractIndexShardComponent { @Override public void run() { try { - if (indexShard.state() == IndexShardState.STARTED) { - indexShard.flush(new Engine.Flush()); - } + indexShard.flush(new Engine.Flush()); } catch (EngineClosedException e) { // we are being closed, ignore } catch (FlushNotAllowedEngineException e) { From 0412e3e2e678ba2c862af154a3c668f08056a780 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 8 Dec 2011 18:08:30 +0200 Subject: [PATCH 013/270] create NodeEnvironment outside of guice so if it fails, it fails early and does not takes ages with guice trying to create it per injection --- .../org/elasticsearch/env/NodeEnvironment.java | 4 ++-- .../elasticsearch/env/NodeEnvironmentModule.java | 14 +++++++++++++- .../elasticsearch/node/internal/InternalNode.java | 4 +++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 37ae247c372..eaee48520ec 100644 --- a/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -48,7 +48,7 @@ public class NodeEnvironment extends AbstractComponent { private final int localNodeId; @Inject - public NodeEnvironment(Settings settings, Environment environment) throws IOException { + public NodeEnvironment(Settings settings, Environment environment) { super(settings); if (!DiscoveryNode.nodeRequiresLocalStorage(settings)) { @@ -117,7 +117,7 @@ public class NodeEnvironment extends AbstractComponent { } } if (locks[0] == null) { - throw new IOException("Failed to obtain node lock on " + Arrays.toString(environment.dataWithClusterFiles()), lastException); + throw new ElasticSearchIllegalStateException("Failed to obtain node lock, is the following location writable?: " + Arrays.toString(environment.dataWithClusterFiles()), lastException); } this.localNodeId = localNodeId; diff --git a/src/main/java/org/elasticsearch/env/NodeEnvironmentModule.java b/src/main/java/org/elasticsearch/env/NodeEnvironmentModule.java index c86d9367e82..d9fe10b5651 100644 --- a/src/main/java/org/elasticsearch/env/NodeEnvironmentModule.java +++ b/src/main/java/org/elasticsearch/env/NodeEnvironmentModule.java @@ -19,6 +19,7 @@ package org.elasticsearch.env; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.AbstractModule; /** @@ -26,11 +27,22 @@ import org.elasticsearch.common.inject.AbstractModule; */ public class NodeEnvironmentModule extends AbstractModule { + private final NodeEnvironment nodeEnvironment; + public NodeEnvironmentModule() { + this(null); + } + + public NodeEnvironmentModule(@Nullable NodeEnvironment nodeEnvironment) { + this.nodeEnvironment = nodeEnvironment; } @Override protected void configure() { - bind(NodeEnvironment.class).asEagerSingleton(); + if (nodeEnvironment != null) { + bind(NodeEnvironment.class).toInstance(nodeEnvironment); + } else { + bind(NodeEnvironment.class).asEagerSingleton(); + } } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/node/internal/InternalNode.java b/src/main/java/org/elasticsearch/node/internal/InternalNode.java index 47405382e54..0dd25a4e27f 100644 --- a/src/main/java/org/elasticsearch/node/internal/InternalNode.java +++ b/src/main/java/org/elasticsearch/node/internal/InternalNode.java @@ -117,6 +117,8 @@ public final class InternalNode implements Node { this.settings = pluginsService.updatedSettings(); this.environment = tuple.v2(); + NodeEnvironment nodeEnvironment = new NodeEnvironment(this.settings, this.environment); + ModulesBuilder modules = new ModulesBuilder(); modules.add(new PluginsModule(settings, pluginsService)); modules.add(new SettingsModule(settings)); @@ -126,7 +128,7 @@ public final class InternalNode implements Node { modules.add(new ScriptModule(settings)); modules.add(new JmxModule(settings)); modules.add(new EnvironmentModule(environment)); - modules.add(new NodeEnvironmentModule()); + modules.add(new NodeEnvironmentModule(nodeEnvironment)); modules.add(new ClusterNameModule(settings)); modules.add(new ThreadPoolModule(settings)); modules.add(new DiscoveryModule(settings)); From a71f2eed99f23ac5c2979b216816c485f6c7b7f3 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 9 Dec 2011 10:04:11 +0200 Subject: [PATCH 014/270] fix test to wait for async indexing to finish --- .../test/integration/aliases/IndexAliasesTests.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.java b/src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.java index 9190b81fa39..e879539564d 100644 --- a/src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.java +++ b/src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.java @@ -45,6 +45,7 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import static com.google.common.collect.Sets.newHashSet; import static org.elasticsearch.client.Requests.*; @@ -516,6 +517,11 @@ public class IndexAliasesTests extends AbstractNodesTests { }); } executor.shutdown(); + boolean done = executor.awaitTermination(10, TimeUnit.SECONDS); + assertThat(done, equalTo(true)); + if (!done) { + executor.shutdownNow(); + } } From 72eb1508c629d525dc97e1351c195ab415bda8fc Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 9 Dec 2011 10:05:07 +0200 Subject: [PATCH 015/270] no need to check and throw eof in readBoolean, already done in readByte --- .../org/elasticsearch/common/io/stream/StreamInput.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index 19fddae5392..1aacc1c787a 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -19,7 +19,6 @@ package org.elasticsearch.common.io.stream; -import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.UTFDataFormatException; @@ -224,10 +223,7 @@ public abstract class StreamInput extends InputStream { * Reads a boolean. */ public final boolean readBoolean() throws IOException { - byte ch = readByte(); - if (ch < 0) - throw new EOFException(); - return (ch != 0); + return readByte() != 0; } From 2b388cff5af52e9890f0ad8010ccbdaad835be96 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 9 Dec 2011 23:50:46 +0200 Subject: [PATCH 016/270] better debug logging of where the default mapping is loaded from --- .../java/org/elasticsearch/index/mapper/MapperService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 1a1b6facaa6..77228d16fa1 100644 --- a/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import com.google.common.collect.UnmodifiableIterator; @@ -125,12 +126,12 @@ public class MapperService extends AbstractIndexComponent implements Iterable Date: Sat, 10 Dec 2011 00:12:42 +0200 Subject: [PATCH 017/270] fix package location of trove extensions --- .../search/type/TransportSearchDfsQueryThenFetchAction.java | 2 +- .../search/type/TransportSearchQueryThenFetchAction.java | 2 +- .../type/TransportSearchScrollQueryThenFetchAction.java | 2 +- .../action/search/type/TransportSearchTypeAction.java | 2 +- src/main/java/org/elasticsearch/common/CacheRecycler.java | 6 +++--- .../elasticsearch/common/trove/ExtTDoubleObjectHashMap.java | 2 +- .../java/org/elasticsearch/common/trove/ExtTHashMap.java | 2 +- .../org/elasticsearch/common/trove/ExtTIntArrayList.java | 2 +- .../elasticsearch/common/trove/ExtTLongObjectHashMap.java | 2 +- .../org/elasticsearch/common/trove/ExtTObjectIntHasMap.java | 2 +- .../common/trove/StringIdentityHashingStrategy.java | 2 +- .../elasticsearch/index/cache/id/simple/SimpleIdCache.java | 2 +- .../index/cache/id/simple/SimpleIdReaderTypeCache.java | 2 +- .../search/controller/SearchPhaseController.java | 2 +- .../java/org/elasticsearch/search/dfs/AggregatedDfs.java | 2 +- .../facet/datehistogram/InternalFullDateHistogramFacet.java | 4 ++-- .../datehistogram/ValueDateHistogramFacetCollector.java | 4 +--- .../ValueScriptDateHistogramFacetCollector.java | 4 +--- .../histogram/unbounded/FullHistogramFacetCollector.java | 4 +--- .../histogram/unbounded/InternalFullHistogramFacet.java | 4 ++-- .../histogram/unbounded/ScriptHistogramFacetCollector.java | 2 +- .../histogram/unbounded/ValueHistogramFacetCollector.java | 4 +--- .../unbounded/ValueScriptHistogramFacetCollector.java | 4 +--- .../termsstats/doubles/InternalTermsStatsDoubleFacet.java | 3 +-- .../termsstats/doubles/TermsStatsDoubleFacetCollector.java | 2 +- .../facet/termsstats/longs/InternalTermsStatsLongFacet.java | 4 ++-- .../termsstats/longs/TermsStatsLongFacetCollector.java | 2 +- .../termsstats/strings/InternalTermsStatsStringFacet.java | 2 +- .../termsstats/strings/TermsStatsStringFacetCollector.java | 2 +- .../org/elasticsearch/search/fetch/FetchSearchRequest.java | 2 +- .../benchmark/trove/StringMapAdjustOrPutBenchmark.java | 2 +- .../search/embedded/SingleShardEmbeddedSearchTests.java | 2 +- .../search/embedded/ThreeShardsEmbeddedSearchTests.java | 2 +- .../ThreeShardsUnbalancedShardsEmbeddedSearchTests.java | 2 +- 34 files changed, 39 insertions(+), 50 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryThenFetchAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryThenFetchAction.java index 9a83b5b4138..4eb0ecb0a88 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryThenFetchAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryThenFetchAction.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.search.type; -import gnu.trove.ExtTIntArrayList; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.*; import org.elasticsearch.cluster.ClusterService; @@ -27,6 +26,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.trove.ExtTIntArrayList; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.action.SearchServiceListener; import org.elasticsearch.search.action.SearchServiceTransportAction; diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchQueryThenFetchAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchQueryThenFetchAction.java index 18d7b9183a2..6ba16d447f1 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchQueryThenFetchAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchQueryThenFetchAction.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.search.type; -import gnu.trove.ExtTIntArrayList; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.*; import org.elasticsearch.cluster.ClusterService; @@ -27,6 +26,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.trove.ExtTIntArrayList; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.action.SearchServiceListener; import org.elasticsearch.search.action.SearchServiceTransportAction; diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java index 0be070da77b..928a737f89f 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.search.type; -import gnu.trove.ExtTIntArrayList; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.*; import org.elasticsearch.cluster.ClusterService; @@ -29,6 +28,7 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.trove.ExtTIntArrayList; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.action.SearchServiceListener; import org.elasticsearch.search.action.SearchServiceTransportAction; diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java index 0f94415741b..fd610cf1edf 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.search.type; -import gnu.trove.ExtTIntArrayList; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.*; import org.elasticsearch.action.support.BaseAction; @@ -33,6 +32,7 @@ import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.trove.ExtTIntArrayList; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.action.SearchServiceListener; diff --git a/src/main/java/org/elasticsearch/common/CacheRecycler.java b/src/main/java/org/elasticsearch/common/CacheRecycler.java index ef65615b634..fd712e922f9 100644 --- a/src/main/java/org/elasticsearch/common/CacheRecycler.java +++ b/src/main/java/org/elasticsearch/common/CacheRecycler.java @@ -19,12 +19,12 @@ package org.elasticsearch.common; -import gnu.trove.ExtTDoubleObjectHashMap; -import gnu.trove.ExtTHashMap; -import gnu.trove.ExtTLongObjectHashMap; import gnu.trove.map.hash.*; import jsr166y.LinkedTransferQueue; import org.elasticsearch.common.compress.lzf.BufferRecycler; +import org.elasticsearch.common.trove.ExtTDoubleObjectHashMap; +import org.elasticsearch.common.trove.ExtTHashMap; +import org.elasticsearch.common.trove.ExtTLongObjectHashMap; import java.lang.ref.SoftReference; import java.util.Arrays; diff --git a/src/main/java/org/elasticsearch/common/trove/ExtTDoubleObjectHashMap.java b/src/main/java/org/elasticsearch/common/trove/ExtTDoubleObjectHashMap.java index bbf13f3881e..95de3531426 100644 --- a/src/main/java/org/elasticsearch/common/trove/ExtTDoubleObjectHashMap.java +++ b/src/main/java/org/elasticsearch/common/trove/ExtTDoubleObjectHashMap.java @@ -17,7 +17,7 @@ * under the License. */ -package gnu.trove; +package org.elasticsearch.common.trove; import gnu.trove.map.TDoubleObjectMap; import gnu.trove.map.hash.TDoubleObjectHashMap; diff --git a/src/main/java/org/elasticsearch/common/trove/ExtTHashMap.java b/src/main/java/org/elasticsearch/common/trove/ExtTHashMap.java index 55a2746eba5..9fca8d6bb52 100644 --- a/src/main/java/org/elasticsearch/common/trove/ExtTHashMap.java +++ b/src/main/java/org/elasticsearch/common/trove/ExtTHashMap.java @@ -17,7 +17,7 @@ * under the License. */ -package gnu.trove; +package org.elasticsearch.common.trove; import gnu.trove.map.hash.THashMap; diff --git a/src/main/java/org/elasticsearch/common/trove/ExtTIntArrayList.java b/src/main/java/org/elasticsearch/common/trove/ExtTIntArrayList.java index 6e3a6061b2c..20229cedd1a 100644 --- a/src/main/java/org/elasticsearch/common/trove/ExtTIntArrayList.java +++ b/src/main/java/org/elasticsearch/common/trove/ExtTIntArrayList.java @@ -17,7 +17,7 @@ * under the License. */ -package gnu.trove; +package org.elasticsearch.common.trove; import gnu.trove.list.array.TIntArrayList; diff --git a/src/main/java/org/elasticsearch/common/trove/ExtTLongObjectHashMap.java b/src/main/java/org/elasticsearch/common/trove/ExtTLongObjectHashMap.java index d95954ea19d..994585f48b5 100644 --- a/src/main/java/org/elasticsearch/common/trove/ExtTLongObjectHashMap.java +++ b/src/main/java/org/elasticsearch/common/trove/ExtTLongObjectHashMap.java @@ -17,7 +17,7 @@ * under the License. */ -package gnu.trove; +package org.elasticsearch.common.trove; import gnu.trove.map.TLongObjectMap; import gnu.trove.map.hash.TLongObjectHashMap; diff --git a/src/main/java/org/elasticsearch/common/trove/ExtTObjectIntHasMap.java b/src/main/java/org/elasticsearch/common/trove/ExtTObjectIntHasMap.java index 7b00aff7d31..1538479bd3b 100644 --- a/src/main/java/org/elasticsearch/common/trove/ExtTObjectIntHasMap.java +++ b/src/main/java/org/elasticsearch/common/trove/ExtTObjectIntHasMap.java @@ -17,7 +17,7 @@ * under the License. */ -package gnu.trove; +package org.elasticsearch.common.trove; import gnu.trove.map.hash.TObjectIntHashMap; diff --git a/src/main/java/org/elasticsearch/common/trove/StringIdentityHashingStrategy.java b/src/main/java/org/elasticsearch/common/trove/StringIdentityHashingStrategy.java index da6e57bf71f..93de9ea80f9 100644 --- a/src/main/java/org/elasticsearch/common/trove/StringIdentityHashingStrategy.java +++ b/src/main/java/org/elasticsearch/common/trove/StringIdentityHashingStrategy.java @@ -17,7 +17,7 @@ * under the License. */ -package gnu.trove; +package org.elasticsearch.common.trove; import gnu.trove.strategy.HashingStrategy; diff --git a/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java b/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java index 26a6d98c908..fa432fc76ff 100644 --- a/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java +++ b/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.cache.id.simple; -import gnu.trove.ExtTObjectIntHasMap; import gnu.trove.impl.Constants; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; @@ -31,6 +30,7 @@ import org.elasticsearch.common.BytesWrap; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.trove.ExtTObjectIntHasMap; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.Index; diff --git a/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdReaderTypeCache.java b/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdReaderTypeCache.java index 90121bf0773..c94242d4995 100644 --- a/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdReaderTypeCache.java +++ b/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdReaderTypeCache.java @@ -19,8 +19,8 @@ package org.elasticsearch.index.cache.id.simple; -import gnu.trove.ExtTObjectIntHasMap; import org.elasticsearch.common.BytesWrap; +import org.elasticsearch.common.trove.ExtTObjectIntHasMap; import org.elasticsearch.index.cache.id.IdReaderTypeCache; /** diff --git a/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java b/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java index 3251ecc1479..910bbffe599 100644 --- a/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java +++ b/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java @@ -23,7 +23,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Ordering; -import gnu.trove.ExtTIntArrayList; import gnu.trove.impl.Constants; import gnu.trove.map.hash.TObjectIntHashMap; import org.apache.lucene.index.Term; @@ -33,6 +32,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.trove.ExtTIntArrayList; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.dfs.AggregatedDfs; import org.elasticsearch.search.dfs.DfsSearchResult; diff --git a/src/main/java/org/elasticsearch/search/dfs/AggregatedDfs.java b/src/main/java/org/elasticsearch/search/dfs/AggregatedDfs.java index 72fbdb702bc..eea53d483b3 100644 --- a/src/main/java/org/elasticsearch/search/dfs/AggregatedDfs.java +++ b/src/main/java/org/elasticsearch/search/dfs/AggregatedDfs.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.dfs; -import gnu.trove.ExtTObjectIntHasMap; import gnu.trove.impl.Constants; import gnu.trove.iterator.TObjectIntIterator; import gnu.trove.map.hash.TObjectIntHashMap; @@ -27,6 +26,7 @@ import org.apache.lucene.index.Term; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.trove.ExtTObjectIntHasMap; import java.io.IOException; diff --git a/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalFullDateHistogramFacet.java b/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalFullDateHistogramFacet.java index d94a1462ac5..9c47b225f3f 100644 --- a/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalFullDateHistogramFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalFullDateHistogramFacet.java @@ -19,10 +19,10 @@ package org.elasticsearch.search.facet.datehistogram; -import gnu.trove.ExtTLongObjectHashMap; import org.elasticsearch.common.CacheRecycler; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.trove.ExtTLongObjectHashMap; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.search.facet.Facet; @@ -152,7 +152,7 @@ public class InternalFullDateHistogramFacet extends InternalDateHistogramFacet { private ComparatorType comparatorType; - ExtTLongObjectHashMap tEntries; + ExtTLongObjectHashMap tEntries; boolean cachedEntries; Collection entries; diff --git a/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueDateHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueDateHistogramFacetCollector.java index a42016fde4f..a29baad629d 100644 --- a/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueDateHistogramFacetCollector.java +++ b/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueDateHistogramFacetCollector.java @@ -19,9 +19,9 @@ package org.elasticsearch.search.facet.datehistogram; -import gnu.trove.ExtTLongObjectHashMap; import org.apache.lucene.index.IndexReader; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.trove.ExtTLongObjectHashMap; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; import org.elasticsearch.index.field.data.NumericFieldData; @@ -38,8 +38,6 @@ import java.io.IOException; /** * A histogram facet collector that uses different fields for the key and the value. - * - * */ public class ValueDateHistogramFacetCollector extends AbstractFacetCollector { diff --git a/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java index d89afdf1651..cd0ce4515bd 100644 --- a/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java +++ b/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java @@ -19,10 +19,10 @@ package org.elasticsearch.search.facet.datehistogram; -import gnu.trove.ExtTLongObjectHashMap; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Scorer; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.trove.ExtTLongObjectHashMap; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; import org.elasticsearch.index.field.data.longs.LongFieldData; @@ -41,8 +41,6 @@ import java.util.Map; /** * A histogram facet collector that uses the same field as the key as well as the * value. - * - * */ public class ValueScriptDateHistogramFacetCollector extends AbstractFacetCollector { diff --git a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/FullHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/FullHistogramFacetCollector.java index 7548dbd0b48..9280d886355 100644 --- a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/FullHistogramFacetCollector.java +++ b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/FullHistogramFacetCollector.java @@ -19,9 +19,9 @@ package org.elasticsearch.search.facet.histogram.unbounded; -import gnu.trove.ExtTLongObjectHashMap; import org.apache.lucene.index.IndexReader; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.trove.ExtTLongObjectHashMap; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; import org.elasticsearch.index.field.data.NumericFieldData; @@ -38,8 +38,6 @@ import java.io.IOException; /** * A histogram facet collector that uses the same field as the key as well as the * value. - * - * */ public class FullHistogramFacetCollector extends AbstractFacetCollector { diff --git a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/InternalFullHistogramFacet.java b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/InternalFullHistogramFacet.java index c6740221448..a7f98916b54 100644 --- a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/InternalFullHistogramFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/InternalFullHistogramFacet.java @@ -19,10 +19,10 @@ package org.elasticsearch.search.facet.histogram.unbounded; -import gnu.trove.ExtTLongObjectHashMap; import org.elasticsearch.common.CacheRecycler; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.trove.ExtTLongObjectHashMap; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.search.facet.Facet; @@ -151,7 +151,7 @@ public class InternalFullHistogramFacet extends InternalHistogramFacet { private ComparatorType comparatorType; - ExtTLongObjectHashMap tEntries; + ExtTLongObjectHashMap tEntries; boolean cachedEntries; Collection entries; diff --git a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ScriptHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ScriptHistogramFacetCollector.java index 3bdc1de2956..335864a2c08 100644 --- a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ScriptHistogramFacetCollector.java +++ b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ScriptHistogramFacetCollector.java @@ -19,10 +19,10 @@ package org.elasticsearch.search.facet.histogram.unbounded; -import gnu.trove.ExtTLongObjectHashMap; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Scorer; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.trove.ExtTLongObjectHashMap; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.facet.AbstractFacetCollector; import org.elasticsearch.search.facet.Facet; diff --git a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueHistogramFacetCollector.java index c00b7b2343a..4b6f29360e6 100644 --- a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueHistogramFacetCollector.java +++ b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueHistogramFacetCollector.java @@ -19,9 +19,9 @@ package org.elasticsearch.search.facet.histogram.unbounded; -import gnu.trove.ExtTLongObjectHashMap; import org.apache.lucene.index.IndexReader; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.trove.ExtTLongObjectHashMap; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; import org.elasticsearch.index.field.data.NumericFieldData; @@ -36,8 +36,6 @@ import java.io.IOException; /** * A histogram facet collector that uses different fields for the key and the value. - * - * */ public class ValueHistogramFacetCollector extends AbstractFacetCollector { diff --git a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueScriptHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueScriptHistogramFacetCollector.java index 6bfb40fe38d..713160b75ad 100644 --- a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueScriptHistogramFacetCollector.java +++ b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueScriptHistogramFacetCollector.java @@ -19,10 +19,10 @@ package org.elasticsearch.search.facet.histogram.unbounded; -import gnu.trove.ExtTLongObjectHashMap; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Scorer; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.trove.ExtTLongObjectHashMap; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; import org.elasticsearch.index.field.data.NumericFieldData; @@ -41,8 +41,6 @@ import java.util.Map; /** * A histogram facet collector that uses the same field as the key as well as the * value. - * - * */ public class ValueScriptHistogramFacetCollector extends AbstractFacetCollector { diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/InternalTermsStatsDoubleFacet.java b/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/InternalTermsStatsDoubleFacet.java index fe065ab224c..368fdcb63a0 100644 --- a/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/InternalTermsStatsDoubleFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/InternalTermsStatsDoubleFacet.java @@ -20,10 +20,10 @@ package org.elasticsearch.search.facet.termsstats.doubles; import com.google.common.collect.ImmutableList; -import gnu.trove.ExtTDoubleObjectHashMap; import org.elasticsearch.common.CacheRecycler; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.trove.ExtTDoubleObjectHashMap; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.search.facet.Facet; @@ -252,7 +252,6 @@ public class InternalTermsStatsDoubleFacet extends InternalTermsStatsFacet { } int missing = 0; ExtTDoubleObjectHashMap map = CacheRecycler.popDoubleObjectMap(); - map.clear(); for (Facet facet : facets) { InternalTermsStatsDoubleFacet tsFacet = (InternalTermsStatsDoubleFacet) facet; missing += tsFacet.missing; diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/TermsStatsDoubleFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/TermsStatsDoubleFacetCollector.java index 0df7fa31e95..f61ca0d7010 100644 --- a/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/TermsStatsDoubleFacetCollector.java +++ b/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/TermsStatsDoubleFacetCollector.java @@ -21,11 +21,11 @@ package org.elasticsearch.search.facet.termsstats.doubles; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import gnu.trove.ExtTDoubleObjectHashMap; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Scorer; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.trove.ExtTDoubleObjectHashMap; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; import org.elasticsearch.index.field.data.NumericFieldData; diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/longs/InternalTermsStatsLongFacet.java b/src/main/java/org/elasticsearch/search/facet/termsstats/longs/InternalTermsStatsLongFacet.java index 4e83251510a..85b5f8aa9e2 100644 --- a/src/main/java/org/elasticsearch/search/facet/termsstats/longs/InternalTermsStatsLongFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/termsstats/longs/InternalTermsStatsLongFacet.java @@ -20,10 +20,10 @@ package org.elasticsearch.search.facet.termsstats.longs; import com.google.common.collect.ImmutableList; -import gnu.trove.ExtTLongObjectHashMap; import org.elasticsearch.common.CacheRecycler; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.trove.ExtTLongObjectHashMap; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.search.facet.Facet; @@ -67,6 +67,7 @@ public class InternalTermsStatsLongFacet extends InternalTermsStatsFacet { public LongEntry(long term, long count, long totalCount, double total, double min, double max) { this.term = term; this.count = count; + this.totalCount = totalCount; this.total = total; this.min = min; this.max = max; @@ -251,7 +252,6 @@ public class InternalTermsStatsLongFacet extends InternalTermsStatsFacet { } int missing = 0; ExtTLongObjectHashMap map = CacheRecycler.popLongObjectMap(); - map.clear(); for (Facet facet : facets) { InternalTermsStatsLongFacet tsFacet = (InternalTermsStatsLongFacet) facet; missing += tsFacet.missing; diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/longs/TermsStatsLongFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/termsstats/longs/TermsStatsLongFacetCollector.java index a6097882210..a0b870427d3 100644 --- a/src/main/java/org/elasticsearch/search/facet/termsstats/longs/TermsStatsLongFacetCollector.java +++ b/src/main/java/org/elasticsearch/search/facet/termsstats/longs/TermsStatsLongFacetCollector.java @@ -21,11 +21,11 @@ package org.elasticsearch.search.facet.termsstats.longs; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import gnu.trove.ExtTLongObjectHashMap; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Scorer; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.trove.ExtTLongObjectHashMap; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; import org.elasticsearch.index.field.data.NumericFieldData; diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java b/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java index fd690291460..942cc879276 100644 --- a/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java @@ -20,10 +20,10 @@ package org.elasticsearch.search.facet.termsstats.strings; import com.google.common.collect.ImmutableList; -import gnu.trove.ExtTHashMap; import org.elasticsearch.common.CacheRecycler; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.trove.ExtTHashMap; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.search.facet.Facet; diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetCollector.java index 21a52e7f292..d711e0d7e43 100644 --- a/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetCollector.java +++ b/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetCollector.java @@ -21,11 +21,11 @@ package org.elasticsearch.search.facet.termsstats.strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import gnu.trove.ExtTHashMap; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Scorer; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.trove.ExtTHashMap; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldData; import org.elasticsearch.index.field.data.FieldDataType; diff --git a/src/main/java/org/elasticsearch/search/fetch/FetchSearchRequest.java b/src/main/java/org/elasticsearch/search/fetch/FetchSearchRequest.java index ecc153d9f64..ed33703ed07 100644 --- a/src/main/java/org/elasticsearch/search/fetch/FetchSearchRequest.java +++ b/src/main/java/org/elasticsearch/search/fetch/FetchSearchRequest.java @@ -19,10 +19,10 @@ package org.elasticsearch.search.fetch; -import gnu.trove.ExtTIntArrayList; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.trove.ExtTIntArrayList; import java.io.IOException; diff --git a/src/test/java/org/elasticsearch/benchmark/trove/StringMapAdjustOrPutBenchmark.java b/src/test/java/org/elasticsearch/benchmark/trove/StringMapAdjustOrPutBenchmark.java index 12b30617a3b..af246ddb947 100644 --- a/src/test/java/org/elasticsearch/benchmark/trove/StringMapAdjustOrPutBenchmark.java +++ b/src/test/java/org/elasticsearch/benchmark/trove/StringMapAdjustOrPutBenchmark.java @@ -19,7 +19,6 @@ package org.elasticsearch.benchmark.trove; -import gnu.trove.StringIdentityHashingStrategy; import gnu.trove.map.custom_hash.TObjectIntCustomHashMap; import gnu.trove.map.hash.THashMap; import gnu.trove.map.hash.TIntIntHashMap; @@ -29,6 +28,7 @@ import gnu.trove.strategy.IdentityHashingStrategy; import jsr166y.ThreadLocalRandom; import org.elasticsearch.common.RandomStringGenerator; import org.elasticsearch.common.StopWatch; +import org.elasticsearch.common.trove.StringIdentityHashingStrategy; import org.elasticsearch.common.unit.SizeValue; import java.util.HashMap; diff --git a/src/test/java/org/elasticsearch/test/integration/search/embedded/SingleShardEmbeddedSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/embedded/SingleShardEmbeddedSearchTests.java index 2a6c7a33926..5ad5237cb32 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/embedded/SingleShardEmbeddedSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/embedded/SingleShardEmbeddedSearchTests.java @@ -21,9 +21,9 @@ package org.elasticsearch.test.integration.search.embedded; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import gnu.trove.ExtTIntArrayList; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Client; +import org.elasticsearch.common.trove.ExtTIntArrayList; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.node.internal.InternalNode; import org.elasticsearch.search.*; diff --git a/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsEmbeddedSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsEmbeddedSearchTests.java index 79270e4028c..3f2fbababe4 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsEmbeddedSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsEmbeddedSearchTests.java @@ -21,13 +21,13 @@ package org.elasticsearch.test.integration.search.embedded; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; -import gnu.trove.ExtTIntArrayList; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.trove.ExtTIntArrayList; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.node.internal.InternalNode; import org.elasticsearch.search.*; diff --git a/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsUnbalancedShardsEmbeddedSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsUnbalancedShardsEmbeddedSearchTests.java index 427cc1ecd25..0ef74c28776 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsUnbalancedShardsEmbeddedSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsUnbalancedShardsEmbeddedSearchTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.test.integration.search.embedded; import com.google.common.collect.ImmutableMap; -import gnu.trove.ExtTIntArrayList; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; @@ -34,6 +33,7 @@ import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.trove.ExtTIntArrayList; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.node.internal.InternalNode; import org.elasticsearch.node.settings.NodeSettingsService; From 3a363976edc7ba0e1d27eea33d217502d28e8817 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 10 Dec 2011 01:38:29 +0200 Subject: [PATCH 018/270] don't debug log conflict base failure (version conflict / document exists) --- .../action/bulk/TransportShardBulkAction.java | 15 +++++++++++---- .../TransportShardReplicationOperationAction.java | 7 ++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index d639d1407ca..dbd0ed0df8b 100644 --- a/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -47,6 +47,7 @@ import org.elasticsearch.index.service.IndexService; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.service.IndexShard; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; @@ -55,8 +56,6 @@ import java.io.IOException; /** * Performs the index operation. - * - * */ public class TransportShardBulkAction extends TransportShardReplicationOperationAction { @@ -178,7 +177,11 @@ public class TransportShardBulkAction extends TransportShardReplicationOperation if (retryPrimaryException(e)) { throw (ElasticSearchException) e; } - logger.debug("[{}][{}] failed to bulk item (index) {}", e, shardRequest.request.index(), shardRequest.shardId, indexRequest); + if (e instanceof ElasticSearchException && ((ElasticSearchException) e).status() == RestStatus.CONFLICT) { + logger.trace("[{}][{}] failed to bulk item (index) {}", e, shardRequest.request.index(), shardRequest.shardId, indexRequest); + } else { + logger.debug("[{}][{}] failed to bulk item (index) {}", e, shardRequest.request.index(), shardRequest.shardId, indexRequest); + } responses[i] = new BulkItemResponse(item.id(), indexRequest.opType().toString().toLowerCase(), new BulkItemResponse.Failure(indexRequest.index(), indexRequest.type(), indexRequest.id(), ExceptionsHelper.detailedMessage(e))); // nullify the request so it won't execute on the replicas @@ -200,7 +203,11 @@ public class TransportShardBulkAction extends TransportShardReplicationOperation if (retryPrimaryException(e)) { throw (ElasticSearchException) e; } - logger.debug("[{}][{}] failed to bulk item (delete) {}", e, shardRequest.request.index(), shardRequest.shardId, deleteRequest); + if (e instanceof ElasticSearchException && ((ElasticSearchException) e).status() == RestStatus.CONFLICT) { + logger.trace("[{}][{}] failed to bulk item (delete) {}", e, shardRequest.request.index(), shardRequest.shardId, deleteRequest); + } else { + logger.debug("[{}][{}] failed to bulk item (delete) {}", e, shardRequest.request.index(), shardRequest.shardId, deleteRequest); + } responses[i] = new BulkItemResponse(item.id(), "delete", new BulkItemResponse.Failure(deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), ExceptionsHelper.detailedMessage(e))); // nullify the request so it won't execute on the replicas diff --git a/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java b/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java index 3e7f7dce0e6..1aa6aa36f7c 100644 --- a/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java @@ -46,6 +46,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.node.NodeClosedException; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.*; @@ -503,7 +504,11 @@ public abstract class TransportShardReplicationOperationAction Date: Sun, 11 Dec 2011 16:24:12 +0200 Subject: [PATCH 019/270] Multicast: Add discovery.zen.ping.multicast.send_ping setting to disable sending ping requests while still having multicast enabled, closes #1479. --- .../zen/ping/multicast/MulticastZenPing.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java b/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java index eaf99c34d5e..7878e979bff 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java @@ -75,6 +75,8 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem private final NetworkService networkService; + private final boolean sendPing; + private volatile DiscoveryNodesProvider nodesProvider; @@ -113,6 +115,8 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem this.bufferSize = componentSettings.getAsInt("buffer_size", 2048); this.ttl = componentSettings.getAsInt("ttl", 3); + this.sendPing = componentSettings.getAsBoolean("send_ping", true); + logger.debug("using group [{}], with port [{}], ttl [{}], and address [{}]", group, port, ttl, address); this.transportService.registerHandler(MulticastPingResponseRequestHandler.ACTION, new MulticastPingResponseRequestHandler()); @@ -221,6 +225,15 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem @Override public void ping(final PingListener listener, final TimeValue timeout) { + if (!sendPing) { + threadPool.cached().execute(new Runnable() { + @Override + public void run() { + listener.onPing(new PingResponse[0]); + } + }); + return; + } final int id = pingIdGenerator.incrementAndGet(); receivedResponses.put(id, new ConcurrentHashMap()); sendPingRequest(id, true); From de861d6f43f0dc41290aa6bcbe0c658a342c3541 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 11 Dec 2011 18:54:07 +0200 Subject: [PATCH 020/270] Support Multicast discovery for external clients, closes #1532. --- .../admin/cluster/node/info/NodeInfo.java | 24 +- .../discovery/zen/DiscoveryNodesProvider.java | 5 + .../discovery/zen/ZenDiscovery.java | 14 ++ .../zen/ping/multicast/MulticastZenPing.java | 223 ++++++++++++------ .../node/service/NodeService.java | 29 ++- .../node/info/RestNodesInfoAction.java | 7 +- .../ping/multicast/MulticastZenPingTests.java | 59 +++++ .../zen/ping/unicast/UnicastZenPingTests.java | 11 + 8 files changed, 283 insertions(+), 89 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java index 526675f8cfb..58fde8e2e5d 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java @@ -39,12 +39,10 @@ import java.util.Map; /** * Node information (static, does not change over time). - * - * */ public class NodeInfo extends NodeOperationResponse { - private ImmutableMap attributes; + private ImmutableMap serviceAttributes; private Settings settings; @@ -63,11 +61,11 @@ public class NodeInfo extends NodeOperationResponse { NodeInfo() { } - public NodeInfo(DiscoveryNode node, ImmutableMap attributes, Settings settings, + public NodeInfo(DiscoveryNode node, ImmutableMap serviceAttributes, Settings settings, OsInfo os, ProcessInfo process, JvmInfo jvm, NetworkInfo network, TransportInfo transport, @Nullable HttpInfo http) { super(node); - this.attributes = attributes; + this.serviceAttributes = serviceAttributes; this.settings = settings; this.os = os; this.process = process; @@ -78,17 +76,17 @@ public class NodeInfo extends NodeOperationResponse { } /** - * The attributes of the node. + * The service attributes of the node. */ - public ImmutableMap attributes() { - return this.attributes; + public ImmutableMap serviceAttributes() { + return this.serviceAttributes; } /** * The attributes of the node. */ - public ImmutableMap getAttributes() { - return attributes(); + public ImmutableMap getServiceAttributes() { + return serviceAttributes(); } /** @@ -191,7 +189,7 @@ public class NodeInfo extends NodeOperationResponse { for (int i = 0; i < size; i++) { builder.put(in.readUTF(), in.readUTF()); } - attributes = builder.build(); + serviceAttributes = builder.build(); settings = ImmutableSettings.readSettingsFromStream(in); if (in.readBoolean()) { os = OsInfo.readOsInfo(in); @@ -216,8 +214,8 @@ public class NodeInfo extends NodeOperationResponse { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeVInt(attributes.size()); - for (Map.Entry entry : attributes.entrySet()) { + out.writeVInt(serviceAttributes.size()); + for (Map.Entry entry : serviceAttributes.entrySet()) { out.writeUTF(entry.getKey()); out.writeUTF(entry.getValue()); } diff --git a/src/main/java/org/elasticsearch/discovery/zen/DiscoveryNodesProvider.java b/src/main/java/org/elasticsearch/discovery/zen/DiscoveryNodesProvider.java index d2487b65f6d..465d2f6647f 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/DiscoveryNodesProvider.java +++ b/src/main/java/org/elasticsearch/discovery/zen/DiscoveryNodesProvider.java @@ -20,6 +20,8 @@ package org.elasticsearch.discovery.zen; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.node.service.NodeService; /** * @@ -27,4 +29,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; public interface DiscoveryNodesProvider { DiscoveryNodes nodes(); + + @Nullable + NodeService nodeService(); } diff --git a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java index a85b14f282b..a07f3e69b37 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.UUID; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.inject.internal.Nullable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.Discovery; @@ -45,6 +46,7 @@ import org.elasticsearch.discovery.zen.ping.ZenPing; import org.elasticsearch.discovery.zen.ping.ZenPingService; import org.elasticsearch.discovery.zen.publish.PublishClusterStateAction; import org.elasticsearch.gateway.GatewayService; +import org.elasticsearch.node.service.NodeService; import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -106,6 +108,9 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen private final AtomicBoolean initialStateSent = new AtomicBoolean(); + @Nullable + private NodeService nodeService; + @Inject public ZenDiscovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, ClusterService clusterService, NodeSettingsService nodeSettingsService, @@ -137,6 +142,10 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen this.membership = new MembershipAction(settings, transportService, this, new MembershipListener()); } + public void setNodeService(@Nullable NodeService nodeService) { + this.nodeService = nodeService; + } + @Override protected void doStart() throws ElasticSearchException { Map nodeAttributes = discoveryNodeService.buildAttributes(); @@ -227,6 +236,11 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen return newNodesBuilder().put(localNode).localNodeId(localNode.id()).build(); } + @Override + public NodeService nodeService() { + return this.nodeService; + } + @Override public void publish(ClusterState clusterState) { if (!master) { diff --git a/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java b/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java index 7878e979bff..61b7518f878 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java @@ -29,6 +29,9 @@ import org.elasticsearch.common.io.stream.*; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.discovery.zen.DiscoveryNodesProvider; import org.elasticsearch.discovery.zen.ping.ZenPing; import org.elasticsearch.discovery.zen.ping.ZenPingException; @@ -36,10 +39,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.*; import java.io.IOException; -import java.net.DatagramPacket; -import java.net.InetAddress; -import java.net.MulticastSocket; -import java.net.SocketTimeoutException; +import java.net.*; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -75,7 +75,7 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem private final NetworkService networkService; - private final boolean sendPing; + private final boolean pingEnabled; private volatile DiscoveryNodesProvider nodesProvider; @@ -115,7 +115,7 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem this.bufferSize = componentSettings.getAsInt("buffer_size", 2048); this.ttl = componentSettings.getAsInt("ttl", 3); - this.sendPing = componentSettings.getAsBoolean("send_ping", true); + this.pingEnabled = componentSettings.getAsBoolean("ping.enabled", true); logger.debug("using group [{}], with port [{}], ttl [{}], and address [{}]", group, port, ttl, address); @@ -225,7 +225,7 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem @Override public void ping(final PingListener listener, final TimeValue timeout) { - if (!sendPing) { + if (!pingEnabled) { threadPool.cached().execute(new Runnable() { @Override public void run() { @@ -359,9 +359,13 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem public void run() { while (running) { try { - int id; - DiscoveryNode requestingNodeX; - ClusterName clusterName; + int id = -1; + DiscoveryNode requestingNodeX = null; + ClusterName clusterName = null; + + Map externalPingData = null; + XContentType xContentType; + synchronized (receiveMutex) { try { multicastSocket.receive(datagramPacketReceive); @@ -374,73 +378,158 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem continue; } try { - StreamInput input = CachedStreamInput.cachedHandles(new BytesStreamInput(datagramPacketReceive.getData(), datagramPacketReceive.getOffset(), datagramPacketReceive.getLength())); - id = input.readInt(); - clusterName = ClusterName.readClusterName(input); - requestingNodeX = readNode(input); + xContentType = XContentFactory.xContentType(datagramPacketReceive.getData(), datagramPacketReceive.getOffset(), datagramPacketReceive.getLength()); + if (xContentType != null) { + // an external ping + externalPingData = XContentFactory.xContent(xContentType) + .createParser(datagramPacketReceive.getData(), datagramPacketReceive.getOffset(), datagramPacketReceive.getLength()) + .mapAndClose(); + } else { + StreamInput input = CachedStreamInput.cachedHandles(new BytesStreamInput(datagramPacketReceive.getData(), datagramPacketReceive.getOffset(), datagramPacketReceive.getLength())); + id = input.readInt(); + clusterName = ClusterName.readClusterName(input); + requestingNodeX = readNode(input); + } } catch (Exception e) { - logger.warn("failed to read requesting node from {}", e, datagramPacketReceive.getSocketAddress()); + logger.warn("failed to read requesting data from {}", e, datagramPacketReceive.getSocketAddress()); continue; } } - DiscoveryNodes discoveryNodes = nodesProvider.nodes(); - final DiscoveryNode requestingNode = requestingNodeX; - if (requestingNode.id().equals(discoveryNodes.localNodeId())) { - // that's me, ignore - continue; - } - if (!clusterName.equals(MulticastZenPing.this.clusterName)) { - if (logger.isTraceEnabled()) { - logger.trace("[{}] received ping_request from [{}], but wrong cluster_name [{}], expected [{}], ignoring", id, requestingNode, clusterName, MulticastZenPing.this.clusterName); - } - continue; - } - // don't connect between two client nodes, no need for that... - if (!discoveryNodes.localNode().shouldConnectTo(requestingNode)) { - if (logger.isTraceEnabled()) { - logger.trace("[{}] received ping_request from [{}], both are client nodes, ignoring", id, requestingNode, clusterName); - } - continue; - } - final MulticastPingResponse multicastPingResponse = new MulticastPingResponse(); - multicastPingResponse.id = id; - multicastPingResponse.pingResponse = new PingResponse(discoveryNodes.localNode(), discoveryNodes.masterNode(), clusterName); - - if (logger.isTraceEnabled()) { - logger.trace("[{}] received ping_request from [{}], sending {}", id, requestingNode, multicastPingResponse.pingResponse); - } - - if (!transportService.nodeConnected(requestingNode)) { - // do the connect and send on a thread pool - threadPool.cached().execute(new Runnable() { - @Override - public void run() { - // connect to the node if possible - try { - transportService.connectToNode(requestingNode); - transportService.sendRequest(requestingNode, MulticastPingResponseRequestHandler.ACTION, multicastPingResponse, new VoidTransportResponseHandler(ThreadPool.Names.SAME) { - @Override - public void handleException(TransportException exp) { - logger.warn("failed to receive confirmation on sent ping response to [{}]", exp, requestingNode); - } - }); - } catch (Exception e) { - logger.warn("failed to connect to requesting node {}", e, requestingNode); - } - } - }); + if (externalPingData != null) { + handleExternalPingRequest(externalPingData, xContentType, datagramPacketReceive.getSocketAddress()); } else { - transportService.sendRequest(requestingNode, MulticastPingResponseRequestHandler.ACTION, multicastPingResponse, new VoidTransportResponseHandler(ThreadPool.Names.SAME) { - @Override - public void handleException(TransportException exp) { - logger.warn("failed to receive confirmation on sent ping response to [{}]", exp, requestingNode); - } - }); + handleNodePingRequest(id, requestingNodeX, clusterName); } } catch (Exception e) { logger.warn("unexpected exception in multicast receiver", e); } } } + + @SuppressWarnings("unchecked") + private void handleExternalPingRequest(Map externalPingData, XContentType contentType, SocketAddress remoteAddress) { + if (externalPingData.containsKey("response")) { + // ignoring responses sent over the multicast channel + logger.trace("got an external ping response (ignoring) from {}, content {}", remoteAddress, externalPingData); + return; + } + + if (multicastSocket == null) { + logger.debug("can't send ping response, no socket, from {}, content {}", remoteAddress, externalPingData); + return; + } + + Map request = (Map) externalPingData.get("request"); + if (request == null) { + logger.warn("malformed external ping request, no 'request' element from {}, content {}", remoteAddress, externalPingData); + return; + } + + String clusterName = request.containsKey("cluster_name") ? request.get("cluster_name").toString() : request.containsKey("clusterName") ? request.get("clusterName").toString() : null; + if (clusterName == null) { + logger.warn("malformed external ping request, missing 'cluster_name' element within request, from {}, content {}", remoteAddress, externalPingData); + return; + } + + if (!clusterName.equals(MulticastZenPing.this.clusterName.value())) { + logger.trace("got request for cluster_name {}, but our cluster_name is {}, from {}, content {}", clusterName, MulticastZenPing.this.clusterName.value(), remoteAddress, externalPingData); + return; + } + if (logger.isTraceEnabled()) { + logger.trace("got external ping request from {}, content {}", remoteAddress, externalPingData); + } + + try { + DiscoveryNode localNode = nodesProvider.nodes().localNode(); + + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + builder.startObject().startObject("response"); + builder.field("cluster_name", MulticastZenPing.this.clusterName.value()); + builder.field("transport_address", localNode.address().toString()); + + if (nodesProvider.nodeService() != null) { + for (Map.Entry attr : nodesProvider.nodeService().attributes().entrySet()) { + builder.field(attr.getKey(), attr.getValue()); + } + } + + builder.startObject("attributes"); + for (Map.Entry attr : localNode.attributes().entrySet()) { + builder.field(attr.getKey(), attr.getValue()); + } + builder.endObject(); + + builder.endObject().endObject(); + synchronized (sendMutex) { + datagramPacketSend.setData(builder.underlyingBytes(), 0, builder.underlyingBytesLength()); + multicastSocket.send(datagramPacketSend); + if (logger.isTraceEnabled()) { + logger.trace("sending external ping response {}", builder.string()); + } + } + } catch (Exception e) { + logger.warn("failed to send external multicast response", e); + } + } + + private void handleNodePingRequest(int id, DiscoveryNode requestingNodeX, ClusterName clusterName) { + if (!pingEnabled) { + return; + } + DiscoveryNodes discoveryNodes = nodesProvider.nodes(); + final DiscoveryNode requestingNode = requestingNodeX; + if (requestingNode.id().equals(discoveryNodes.localNodeId())) { + // that's me, ignore + return; + } + if (!clusterName.equals(MulticastZenPing.this.clusterName)) { + if (logger.isTraceEnabled()) { + logger.trace("[{}] received ping_request from [{}], but wrong cluster_name [{}], expected [{}], ignoring", id, requestingNode, clusterName, MulticastZenPing.this.clusterName); + } + return; + } + // don't connect between two client nodes, no need for that... + if (!discoveryNodes.localNode().shouldConnectTo(requestingNode)) { + if (logger.isTraceEnabled()) { + logger.trace("[{}] received ping_request from [{}], both are client nodes, ignoring", id, requestingNode, clusterName); + } + return; + } + final MulticastPingResponse multicastPingResponse = new MulticastPingResponse(); + multicastPingResponse.id = id; + multicastPingResponse.pingResponse = new PingResponse(discoveryNodes.localNode(), discoveryNodes.masterNode(), clusterName); + + if (logger.isTraceEnabled()) { + logger.trace("[{}] received ping_request from [{}], sending {}", id, requestingNode, multicastPingResponse.pingResponse); + } + + if (!transportService.nodeConnected(requestingNode)) { + // do the connect and send on a thread pool + threadPool.cached().execute(new Runnable() { + @Override + public void run() { + // connect to the node if possible + try { + transportService.connectToNode(requestingNode); + transportService.sendRequest(requestingNode, MulticastPingResponseRequestHandler.ACTION, multicastPingResponse, new VoidTransportResponseHandler(ThreadPool.Names.SAME) { + @Override + public void handleException(TransportException exp) { + logger.warn("failed to receive confirmation on sent ping response to [{}]", exp, requestingNode); + } + }); + } catch (Exception e) { + logger.warn("failed to connect to requesting node {}", e, requestingNode); + } + } + }); + } else { + transportService.sendRequest(requestingNode, MulticastPingResponseRequestHandler.ACTION, multicastPingResponse, new VoidTransportResponseHandler(ThreadPool.Names.SAME) { + @Override + public void handleException(TransportException exp) { + logger.warn("failed to receive confirmation on sent ping response to [{}]", exp, requestingNode); + } + }); + } + } } } diff --git a/src/main/java/org/elasticsearch/node/service/NodeService.java b/src/main/java/org/elasticsearch/node/service/NodeService.java index 28537bfed74..3728e450f9c 100644 --- a/src/main/java/org/elasticsearch/node/service/NodeService.java +++ b/src/main/java/org/elasticsearch/node/service/NodeService.java @@ -48,7 +48,7 @@ public class NodeService extends AbstractComponent { @Nullable private HttpServer httpServer; - private volatile ImmutableMap nodeAttributes = ImmutableMap.of(); + private volatile ImmutableMap serviceAttributes = ImmutableMap.of(); @Inject public NodeService(Settings settings, MonitorService monitorService, ClusterService clusterService, TransportService transportService, IndicesService indicesService) { @@ -63,16 +63,33 @@ public class NodeService extends AbstractComponent { this.httpServer = httpServer; } - public synchronized void putNodeAttribute(String key, String value) { - nodeAttributes = new MapBuilder().putAll(nodeAttributes).put(key, value).immutableMap(); + @Deprecated + public void putNodeAttribute(String key, String value) { + putAttribute(key, value); } - public synchronized void removeNodeAttribute(String key) { - nodeAttributes = new MapBuilder().putAll(nodeAttributes).remove(key).immutableMap(); + @Deprecated + public void removeNodeAttribute(String key) { + removeAttribute(key); + } + + public synchronized void putAttribute(String key, String value) { + serviceAttributes = new MapBuilder().putAll(serviceAttributes).put(key, value).immutableMap(); + } + + public synchronized void removeAttribute(String key) { + serviceAttributes = new MapBuilder().putAll(serviceAttributes).remove(key).immutableMap(); + } + + /** + * Attributes different services in the node can add to be reported as part of the node info (for example). + */ + public ImmutableMap attributes() { + return this.serviceAttributes; } public NodeInfo info() { - return new NodeInfo(clusterService.state().nodes().localNode(), nodeAttributes, settings, + return new NodeInfo(clusterService.state().nodes().localNode(), serviceAttributes, settings, monitorService.osService().info(), monitorService.processService().info(), monitorService.jvmService().info(), monitorService.networkService().info(), transportService.info(), httpServer == null ? null : httpServer.info()); diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java index 20f46ba31f6..184e3ed9c6c 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java @@ -73,15 +73,16 @@ public class RestNodesInfoAction extends BaseRestHandler { builder.field("name", nodeInfo.node().name(), XContentBuilder.FieldCaseConversion.NONE); builder.field("transport_address", nodeInfo.node().address().toString()); + for (Map.Entry nodeAttribute : nodeInfo.serviceAttributes().entrySet()) { + builder.field(nodeAttribute.getKey(), nodeAttribute.getValue()); + } + builder.startObject("attributes"); for (Map.Entry attr : nodeInfo.node().attributes().entrySet()) { builder.field(attr.getKey(), attr.getValue()); } builder.endObject(); - for (Map.Entry nodeAttribute : nodeInfo.attributes().entrySet()) { - builder.field(nodeAttribute.getKey(), nodeAttribute.getValue()); - } if (includeSettings) { builder.startObject("settings"); diff --git a/src/test/java/org/elasticsearch/test/unit/discovery/zen/ping/multicast/MulticastZenPingTests.java b/src/test/java/org/elasticsearch/test/unit/discovery/zen/ping/multicast/MulticastZenPingTests.java index 9c2cce4476d..3b3c47e9276 100644 --- a/src/test/java/org/elasticsearch/test/unit/discovery/zen/ping/multicast/MulticastZenPingTests.java +++ b/src/test/java/org/elasticsearch/test/unit/discovery/zen/ping/multicast/MulticastZenPingTests.java @@ -22,15 +22,23 @@ package org.elasticsearch.test.unit.discovery.zen.ping.multicast; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.discovery.zen.DiscoveryNodesProvider; import org.elasticsearch.discovery.zen.ping.ZenPing; import org.elasticsearch.discovery.zen.ping.multicast.MulticastZenPing; +import org.elasticsearch.node.service.NodeService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.local.LocalTransport; import org.testng.annotations.Test; +import java.net.DatagramPacket; +import java.net.InetAddress; +import java.net.MulticastSocket; + import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -56,6 +64,11 @@ public class MulticastZenPingTests { public DiscoveryNodes nodes() { return DiscoveryNodes.newNodesBuilder().put(nodeA).localNodeId("A").build(); } + + @Override + public NodeService nodeService() { + return null; + } }); zenPingA.start(); @@ -65,6 +78,11 @@ public class MulticastZenPingTests { public DiscoveryNodes nodes() { return DiscoveryNodes.newNodesBuilder().put(nodeB).localNodeId("B").build(); } + + @Override + public NodeService nodeService() { + return null; + } }); zenPingB.start(); @@ -80,4 +98,45 @@ public class MulticastZenPingTests { threadPool.shutdown(); } } + + @Test + public void testExternalPing() throws Exception { + ThreadPool threadPool = new ThreadPool(); + + ClusterName clusterName = new ClusterName("test"); + final TransportService transportServiceA = new TransportService(new LocalTransport(threadPool), threadPool).start(); + final DiscoveryNode nodeA = new DiscoveryNode("A", transportServiceA.boundAddress().publishAddress()); + + MulticastZenPing zenPingA = new MulticastZenPing(threadPool, transportServiceA, clusterName); + zenPingA.setNodesProvider(new DiscoveryNodesProvider() { + @Override + public DiscoveryNodes nodes() { + return DiscoveryNodes.newNodesBuilder().put(nodeA).localNodeId("A").build(); + } + + @Override + public NodeService nodeService() { + return null; + } + }); + zenPingA.start(); + + try { + Loggers.getLogger(MulticastZenPing.class).setLevel("TRACE"); + MulticastSocket multicastSocket = new MulticastSocket(54328); + multicastSocket.setReceiveBufferSize(2048); + multicastSocket.setSendBufferSize(2048); + multicastSocket.setSoTimeout(60000); + + DatagramPacket datagramPacket = new DatagramPacket(new byte[2048], 2048, InetAddress.getByName("224.2.2.4"), 54328); + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("request").field("cluster_name", "test").endObject().endObject(); + datagramPacket.setData(builder.copiedBytes()); + multicastSocket.send(datagramPacket); + Thread.sleep(100); + } finally { + Loggers.getLogger(MulticastZenPing.class).setLevel("INFO"); + zenPingA.close(); + threadPool.shutdown(); + } + } } diff --git a/src/test/java/org/elasticsearch/test/unit/discovery/zen/ping/unicast/UnicastZenPingTests.java b/src/test/java/org/elasticsearch/test/unit/discovery/zen/ping/unicast/UnicastZenPingTests.java index eedf50d39d0..ebeb09e8776 100644 --- a/src/test/java/org/elasticsearch/test/unit/discovery/zen/ping/unicast/UnicastZenPingTests.java +++ b/src/test/java/org/elasticsearch/test/unit/discovery/zen/ping/unicast/UnicastZenPingTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.zen.DiscoveryNodesProvider; import org.elasticsearch.discovery.zen.ping.ZenPing; import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing; +import org.elasticsearch.node.service.NodeService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.netty.NettyTransport; @@ -69,6 +70,11 @@ public class UnicastZenPingTests { public DiscoveryNodes nodes() { return DiscoveryNodes.newNodesBuilder().put(nodeA).localNodeId("A").build(); } + + @Override + public NodeService nodeService() { + return null; + } }); zenPingA.start(); @@ -78,6 +84,11 @@ public class UnicastZenPingTests { public DiscoveryNodes nodes() { return DiscoveryNodes.newNodesBuilder().put(nodeB).localNodeId("B").build(); } + + @Override + public NodeService nodeService() { + return null; + } }); zenPingB.start(); From b0b379dc88ee517ad7f26353cd55e64601433b16 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 12 Dec 2011 01:09:46 +0200 Subject: [PATCH 021/270] River: When deleting a river, make sure when its closed that its data is deleted, closes #1534. --- .../metadata/MetaDataMappingService.java | 16 +++++++-- .../elasticsearch/river/RiversService.java | 34 ++++++++++++++++++- .../river/routing/RiversRouter.java | 1 - 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java index f2d72137ae3..aff6d5fec4a 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java @@ -235,14 +235,24 @@ public class MetaDataMappingService extends AbstractComponent { throw new IndexMissingException(new Index("_all")); } - logger.info("[{}] remove_mapping [{}]", request.indices, request.mappingType); MetaData.Builder builder = newMetaDataBuilder().metaData(currentState.metaData()); + boolean changed = false; for (String indexName : request.indices) { - if (currentState.metaData().hasIndex(indexName)) { - builder.put(newIndexMetaDataBuilder(currentState.metaData().index(indexName)).removeMapping(request.mappingType)); + IndexMetaData indexMetaData = currentState.metaData().index(indexName); + if (indexMetaData != null) { + if (indexMetaData.mappings().containsKey(request.mappingType)) { + builder.put(newIndexMetaDataBuilder(indexMetaData).removeMapping(request.mappingType)); + changed = true; + } } } + if (!changed) { + return currentState; + } + + logger.info("[{}] remove_mapping [{}]", request.indices, request.mappingType); + return ClusterState.builder().state(currentState).metaData(builder).build(); } diff --git a/src/main/java/org/elasticsearch/river/RiversService.java b/src/main/java/org/elasticsearch/river/RiversService.java index 3b9d200d1e4..5da98bacabf 100644 --- a/src/main/java/org/elasticsearch/river/RiversService.java +++ b/src/main/java/org/elasticsearch/river/RiversService.java @@ -27,6 +27,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.NoShardAvailableActionException; import org.elasticsearch.action.WriteConsistencyLevel; +import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterService; @@ -208,11 +209,42 @@ public class RiversService extends AbstractLifecycleComponent { RiverClusterState state = event.state(); // first, go over and delete ones that either don't exists or are not allocated - for (RiverName riverName : rivers.keySet()) { + for (final RiverName riverName : rivers.keySet()) { RiverRouting routing = state.routing().routing(riverName); if (routing == null || !localNode.equals(routing.node())) { // not routed at all, and not allocated here, clean it (we delete the relevant ones before) closeRiver(riverName); + // also, double check and delete the river content if it was deleted (_meta does not exists) + try { + client.prepareGet(riverIndexName, riverName.name(), "_meta").setListenerThreaded(true).execute(new ActionListener() { + @Override + public void onResponse(GetResponse getResponse) { + if (!getResponse.exists()) { + // verify the river is deleted + client.admin().indices().prepareDeleteMapping(riverIndexName).setType(riverName.name()).execute(new ActionListener() { + @Override + public void onResponse(DeleteMappingResponse deleteMappingResponse) { + // all is well... + } + + @Override + public void onFailure(Throwable e) { + logger.debug("failed to (double) delete river [{}] content", e, riverName.name()); + } + }); + } + } + + @Override + public void onFailure(Throwable e) { + logger.debug("failed to (double) delete river [{}] content", e, riverName.name()); + } + }); + } catch (IndexMissingException e) { + // all is well, the _river index was deleted + } catch (Exception e) { + logger.warn("unexpected failure when trying to verify river [{}] deleted", e, riverName.name()); + } } } diff --git a/src/main/java/org/elasticsearch/river/routing/RiversRouter.java b/src/main/java/org/elasticsearch/river/routing/RiversRouter.java index 9bcc61c10a6..61e5d69923f 100644 --- a/src/main/java/org/elasticsearch/river/routing/RiversRouter.java +++ b/src/main/java/org/elasticsearch/river/routing/RiversRouter.java @@ -106,7 +106,6 @@ public class RiversRouter extends AbstractLifecycleComponent imple if (!currentState.routing().hasRiverByName(mappingType)) { // no river, we need to add it to the routing with no node allocation try { - client.admin().indices().prepareRefresh(riverIndexName).execute().actionGet(); GetResponse getResponse = client.prepareGet(riverIndexName, mappingType, "_meta").execute().actionGet(); if (getResponse.exists()) { String riverType = XContentMapValues.nodeStringValue(getResponse.sourceAsMap().get("type"), null); From 775339d78a55b84799b823e23bed38b8f10d794f Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 12 Dec 2011 15:04:54 +0200 Subject: [PATCH 022/270] Allow to configure a shard with no file based data location locking, closes #1535. --- .../org/elasticsearch/index/store/fs/FsDirectoryService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/index/store/fs/FsDirectoryService.java b/src/main/java/org/elasticsearch/index/store/fs/FsDirectoryService.java index 9264e17d748..e475ee099dc 100644 --- a/src/main/java/org/elasticsearch/index/store/fs/FsDirectoryService.java +++ b/src/main/java/org/elasticsearch/index/store/fs/FsDirectoryService.java @@ -45,13 +45,15 @@ public abstract class FsDirectoryService extends AbstractIndexShardComponent imp } protected LockFactory buildLockFactory() throws IOException { - String fsLock = componentSettings.get("fs_lock", "native"); + String fsLock = componentSettings.get("lock", componentSettings.get("fs_lock", "native")); LockFactory lockFactory = NoLockFactory.getNoLockFactory(); if (fsLock.equals("native")) { // TODO LUCENE MONITOR: this is not needed in next Lucene version lockFactory = new NativeFSLockFactory(); } else if (fsLock.equals("simple")) { lockFactory = new SimpleFSLockFactory(); + } else if (fsLock.equals("none")) { + lockFactory = NoLockFactory.getNoLockFactory(); } return lockFactory; } From ce999489d5c8c89146e43593aac1c56743f00214 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 12 Dec 2011 20:22:19 +0200 Subject: [PATCH 023/270] set the node service on the discovery to get the additional service level attributes (issue #1532) --- src/main/java/org/elasticsearch/discovery/Discovery.java | 9 +++++++-- .../elasticsearch/discovery/local/LocalDiscovery.java | 7 +++++++ .../org/elasticsearch/discovery/zen/ZenDiscovery.java | 1 + .../java/org/elasticsearch/node/service/NodeService.java | 4 +++- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/discovery/Discovery.java b/src/main/java/org/elasticsearch/discovery/Discovery.java index 5adce51c7c4..c702d98c4b1 100644 --- a/src/main/java/org/elasticsearch/discovery/Discovery.java +++ b/src/main/java/org/elasticsearch/discovery/Discovery.java @@ -24,13 +24,13 @@ import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.component.LifecycleComponent; +import org.elasticsearch.common.inject.internal.Nullable; +import org.elasticsearch.node.service.NodeService; /** * A pluggable module allowing to implement discovery of other nodes, publishing of the cluster * state to all nodes, electing a master of the cluster that raises cluster state change * events. - * - * */ public interface Discovery extends LifecycleComponent { @@ -44,6 +44,11 @@ public interface Discovery extends LifecycleComponent { String nodeDescription(); + /** + * Here as a hack to solve dep injection problem... + */ + void setNodeService(@Nullable NodeService nodeService); + /** * Publish all the changes to the cluster from the master (can be called just by the master). The publish * process should not publish this state to the master as well! (the master is sending it...). diff --git a/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java index 97bd6d9d46b..e10dd070d88 100644 --- a/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java @@ -29,9 +29,11 @@ import org.elasticsearch.cluster.node.DiscoveryNodeService; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.inject.internal.Nullable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.InitialStateDiscoveryListener; +import org.elasticsearch.node.service.NodeService; import org.elasticsearch.transport.TransportService; import java.util.Queue; @@ -82,6 +84,11 @@ public class LocalDiscovery extends AbstractLifecycleComponent implem this.discoveryNodeService = discoveryNodeService; } + @Override + public void setNodeService(@Nullable NodeService nodeService) { + // nothing to do here + } + @Override protected void doStart() throws ElasticSearchException { synchronized (clusterGroups) { diff --git a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java index a07f3e69b37..efaed31d0cb 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java @@ -142,6 +142,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen this.membership = new MembershipAction(settings, transportService, this, new MembershipListener()); } + @Override public void setNodeService(@Nullable NodeService nodeService) { this.nodeService = nodeService; } diff --git a/src/main/java/org/elasticsearch/node/service/NodeService.java b/src/main/java/org/elasticsearch/node/service/NodeService.java index 3728e450f9c..86cc371799e 100644 --- a/src/main/java/org/elasticsearch/node/service/NodeService.java +++ b/src/main/java/org/elasticsearch/node/service/NodeService.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.discovery.Discovery; import org.elasticsearch.http.HttpServer; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.monitor.MonitorService; @@ -51,12 +52,13 @@ public class NodeService extends AbstractComponent { private volatile ImmutableMap serviceAttributes = ImmutableMap.of(); @Inject - public NodeService(Settings settings, MonitorService monitorService, ClusterService clusterService, TransportService transportService, IndicesService indicesService) { + public NodeService(Settings settings, MonitorService monitorService, Discovery discovery, ClusterService clusterService, TransportService transportService, IndicesService indicesService) { super(settings); this.monitorService = monitorService; this.clusterService = clusterService; this.transportService = transportService; this.indicesService = indicesService; + discovery.setNodeService(this); } public void setHttpServer(@Nullable HttpServer httpServer) { From 5d65b98611c3f3702c75cdd97e82f1670f8848a4 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 12 Dec 2011 20:35:17 +0200 Subject: [PATCH 024/270] add the version number to the json multicast response --- .../discovery/zen/ping/multicast/MulticastZenPing.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java b/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java index 61b7518f878..356c60cc7f8 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java @@ -21,6 +21,7 @@ package org.elasticsearch.discovery.zen.ping.multicast; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ElasticSearchIllegalStateException; +import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -445,6 +446,7 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem XContentBuilder builder = XContentFactory.contentBuilder(contentType); builder.startObject().startObject("response"); builder.field("cluster_name", MulticastZenPing.this.clusterName.value()); + builder.startObject("version").field("number", Version.CURRENT.number()).field("snapshot_build", Version.CURRENT.snapshot).endObject(); builder.field("transport_address", localNode.address().toString()); if (nodesProvider.nodeService() != null) { From 346d2fc28c33c0660d43eff7200b3a3157efdbf4 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 12 Dec 2011 20:53:47 +0200 Subject: [PATCH 025/270] remove unused setting --- .../org/elasticsearch/indices/recovery/RecoverySource.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java b/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java index 81ab3be8339..6efe35773c6 100644 --- a/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java +++ b/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java @@ -23,7 +23,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.lucene.store.IndexInput; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.StopWatch; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; @@ -53,10 +52,6 @@ import java.util.concurrent.atomic.AtomicReference; */ public class RecoverySource extends AbstractComponent { - static { - MetaData.addDynamicSettings("index.shard.recovery.concurrent_streams"); - } - public static class Actions { public static final String START_RECOVERY = "index/shard/recovery/startRecovery"; } From 0d195eb36804639ee22c1d69c9917ec47d85d72a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 12 Dec 2011 20:54:13 +0200 Subject: [PATCH 026/270] move unused variable --- .../org/elasticsearch/indices/recovery/RecoverySource.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java b/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java index 6efe35773c6..67762296cf6 100644 --- a/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java +++ b/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java @@ -56,8 +56,6 @@ public class RecoverySource extends AbstractComponent { public static final String START_RECOVERY = "index/shard/recovery/startRecovery"; } - private final ThreadPool threadPool; - private final TransportService transportService; private final IndicesService indicesService; @@ -66,10 +64,9 @@ public class RecoverySource extends AbstractComponent { @Inject - public RecoverySource(Settings settings, ThreadPool threadPool, TransportService transportService, IndicesService indicesService, + public RecoverySource(Settings settings, TransportService transportService, IndicesService indicesService, RecoverySettings recoverySettings) { super(settings); - this.threadPool = threadPool; this.transportService = transportService; this.indicesService = indicesService; From 0cd38f505def2c99f53feb8203101b2553217277 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 13 Dec 2011 17:47:09 +0200 Subject: [PATCH 027/270] Query DSL: Replace index.query.bool.max_clause_count with indices.query.bool.max_clause_count (old one still supported), closes #1538. --- .../java/org/elasticsearch/index/query/BoolQueryParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java b/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java index eefa2914ab9..7667c3ee13c 100644 --- a/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java @@ -42,7 +42,7 @@ public class BoolQueryParser implements QueryParser { @Inject public BoolQueryParser(Settings settings) { - BooleanQuery.setMaxClauseCount(settings.getAsInt("index.query.bool.max_clause_count", BooleanQuery.getMaxClauseCount())); + BooleanQuery.setMaxClauseCount(settings.getAsInt("index.query.bool.max_clause_count", settings.getAsInt("indices.query.bool.max_clause_count", BooleanQuery.getMaxClauseCount()))); } @Override From 679bd1c877a589cc0cfd93e9f5cceb1506c36398 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 13 Dec 2011 18:38:30 +0200 Subject: [PATCH 028/270] Query DSL: query_string analyze wildcard option with prefix to automatically do OR'ed wildcard when its broken down into several tokens, closes #1539. --- .../apache/lucene/queryParser/MapperQueryParser.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java b/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java index 3bed69cf610..ff4c322aafa 100644 --- a/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java +++ b/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java @@ -50,8 +50,6 @@ import static org.elasticsearch.index.query.support.QueryParsers.wrapSmartNameQu *

*

Also breaks fields with [type].[name] into a boolean query that must include the type * as well as the query on the name. - * - * */ public class MapperQueryParser extends QueryParser { @@ -229,7 +227,15 @@ public class MapperQueryParser extends QueryParser { if (tlist.size() == 1) { return super.getPrefixQuery(field, tlist.get(0)); } else { - return super.getPrefixQuery(field, termStr); + // build a boolean query with prefix on each one... + List clauses = new ArrayList(); + for (String token : tlist) { + clauses.add(new BooleanClause(super.getPrefixQuery(field, token), BooleanClause.Occur.SHOULD)); + } + return getBooleanQuery(clauses, true); + + //return super.getPrefixQuery(field, termStr); + /* this means that the analyzer used either added or consumed * (common for a stemmer) tokens, and we can't build a PrefixQuery */ // throw new ParseException("Cannot build PrefixQuery with analyzer " From d2c1ec5f25e0442458f1e0db6d93eebf198ab222 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 13 Dec 2011 19:02:01 +0200 Subject: [PATCH 029/270] Query DSL: Allow to default certain settings in query_string / field queries, closes #1540. --- .../lucene/queryParser/QueryParserSettings.java | 7 +++++-- .../elasticsearch/index/query/FieldQueryParser.java | 11 ++++++++++- .../index/query/QueryStringQueryParser.java | 11 ++++++++++- .../indices/query/IndicesQueriesRegistry.java | 4 ++-- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/apache/lucene/queryParser/QueryParserSettings.java b/src/main/java/org/apache/lucene/queryParser/QueryParserSettings.java index 825cc0fc4bb..eaef33cd479 100644 --- a/src/main/java/org/apache/lucene/queryParser/QueryParserSettings.java +++ b/src/main/java/org/apache/lucene/queryParser/QueryParserSettings.java @@ -28,18 +28,21 @@ import org.apache.lucene.search.MultiTermQuery; */ public class QueryParserSettings { + public static final boolean DEFAULT_ALLOW_LEADING_WILDCARD = true; + public static final boolean DEFAULT_ANALYZE_WILDCARD = false; + private String queryString; private String defaultField; private float boost = 1.0f; private MapperQueryParser.Operator defaultOperator = QueryParser.Operator.OR; private boolean autoGeneratePhraseQueries = false; - private boolean allowLeadingWildcard = true; + private boolean allowLeadingWildcard = DEFAULT_ALLOW_LEADING_WILDCARD; private boolean lowercaseExpandedTerms = true; private boolean enablePositionIncrements = true; private int phraseSlop = 0; private float fuzzyMinSim = FuzzyQuery.defaultMinSimilarity; private int fuzzyPrefixLength = FuzzyQuery.defaultPrefixLength; - private boolean analyzeWildcard = false; + private boolean analyzeWildcard = DEFAULT_ANALYZE_WILDCARD; private boolean escape = false; private Analyzer analyzer = null; private MultiTermQuery.RewriteMethod rewriteMethod = MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT; diff --git a/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java b/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java index 4c5a768100b..2ba602ddcfa 100644 --- a/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.support.QueryParsers; @@ -41,8 +42,13 @@ public class FieldQueryParser implements QueryParser { public static final String NAME = "field"; + private final boolean defaultAnalyzeWildcard; + private final boolean defaultAllowLeadingWildcard; + @Inject - public FieldQueryParser() { + public FieldQueryParser(Settings settings) { + this.defaultAnalyzeWildcard = settings.getAsBoolean("indices.query.query_string.analyze_wildcard", QueryParserSettings.DEFAULT_ANALYZE_WILDCARD); + this.defaultAllowLeadingWildcard = settings.getAsBoolean("indices.query.query_string.allowLeadingWildcard", QueryParserSettings.DEFAULT_ALLOW_LEADING_WILDCARD); } @Override @@ -60,6 +66,9 @@ public class FieldQueryParser implements QueryParser { QueryParserSettings qpSettings = new QueryParserSettings(); qpSettings.defaultField(fieldName); + qpSettings.analyzeWildcard(defaultAnalyzeWildcard); + qpSettings.allowLeadingWildcard(defaultAllowLeadingWildcard); + token = parser.nextToken(); if (token == XContentParser.Token.START_OBJECT) { String currentFieldName = null; diff --git a/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java b/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java index ff2df060ed4..7e2d7bd94bf 100644 --- a/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java @@ -25,12 +25,14 @@ import gnu.trove.map.hash.TObjectFloatHashMap; import org.apache.lucene.queryParser.MapperQueryParser; import org.apache.lucene.queryParser.MultiFieldQueryParserSettings; import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.queryParser.QueryParserSettings; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.internal.AllFieldMapper; import org.elasticsearch.index.query.support.QueryParsers; @@ -47,8 +49,13 @@ public class QueryStringQueryParser implements QueryParser { public static final String NAME = "query_string"; + private final boolean defaultAnalyzeWildcard; + private final boolean defaultAllowLeadingWildcard; + @Inject - public QueryStringQueryParser() { + public QueryStringQueryParser(Settings settings) { + this.defaultAnalyzeWildcard = settings.getAsBoolean("indices.query.query_string.analyze_wildcard", QueryParserSettings.DEFAULT_ANALYZE_WILDCARD); + this.defaultAllowLeadingWildcard = settings.getAsBoolean("indices.query.query_string.allowLeadingWildcard", QueryParserSettings.DEFAULT_ALLOW_LEADING_WILDCARD); } @Override @@ -62,6 +69,8 @@ public class QueryStringQueryParser implements QueryParser { MultiFieldQueryParserSettings qpSettings = new MultiFieldQueryParserSettings(); qpSettings.defaultField(AllFieldMapper.NAME); + qpSettings.analyzeWildcard(defaultAnalyzeWildcard); + qpSettings.allowLeadingWildcard(defaultAllowLeadingWildcard); String currentFieldName = null; XContentParser.Token token; diff --git a/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java b/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java index 61f9b337a60..ccc0154f28b 100644 --- a/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java +++ b/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java @@ -45,13 +45,13 @@ public class IndicesQueriesRegistry { addQueryParser(queryParsers, new DisMaxQueryParser()); addQueryParser(queryParsers, new IdsQueryParser()); addQueryParser(queryParsers, new MatchAllQueryParser()); - addQueryParser(queryParsers, new QueryStringQueryParser()); + addQueryParser(queryParsers, new QueryStringQueryParser(settings)); addQueryParser(queryParsers, new BoostingQueryParser()); addQueryParser(queryParsers, new BoolQueryParser(settings)); addQueryParser(queryParsers, new TermQueryParser()); addQueryParser(queryParsers, new TermsQueryParser()); addQueryParser(queryParsers, new FuzzyQueryParser()); - addQueryParser(queryParsers, new FieldQueryParser()); + addQueryParser(queryParsers, new FieldQueryParser(settings)); addQueryParser(queryParsers, new RangeQueryParser()); addQueryParser(queryParsers, new PrefixQueryParser()); addQueryParser(queryParsers, new WildcardQueryParser()); From 41a9dc1e6ddc75a253b81748c33e8eebc55a9b24 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 13 Dec 2011 19:03:32 +0200 Subject: [PATCH 030/270] make sigar optional --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index b671570df2b..1d8b395367b 100644 --- a/pom.xml +++ b/pom.xml @@ -210,6 +210,7 @@ 1.6.4 system ${basedir}/lib/sigar/sigar-1.6.4.jar + true From 4693b1e875ed673dfb9b875303b764fb7026d2e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Laleve=CC=81e?= Date: Mon, 31 Oct 2011 14:50:18 +0100 Subject: [PATCH 031/270] Add the hostname to the node info --- .../admin/cluster/node/info/NodeInfo.java | 31 ++++++++++++++++++- .../node/service/NodeService.java | 12 ++++++- .../node/info/RestNodesInfoAction.java | 4 +++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java index 58fde8e2e5d..d53a52bce46 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java @@ -44,6 +44,9 @@ public class NodeInfo extends NodeOperationResponse { private ImmutableMap serviceAttributes; + @Nullable + private String hostname; + private Settings settings; private OsInfo os; @@ -61,10 +64,11 @@ public class NodeInfo extends NodeOperationResponse { NodeInfo() { } - public NodeInfo(DiscoveryNode node, ImmutableMap serviceAttributes, Settings settings, + public NodeInfo(String hostname, DiscoveryNode node, ImmutableMap serviceAttributes, Settings settings, OsInfo os, ProcessInfo process, JvmInfo jvm, NetworkInfo network, TransportInfo transport, @Nullable HttpInfo http) { super(node); + this.hostname = hostname; this.serviceAttributes = serviceAttributes; this.settings = settings; this.os = os; @@ -75,6 +79,22 @@ public class NodeInfo extends NodeOperationResponse { this.http = http; } + /** + * System's hostname. null in case of UnknownHostException + */ + @Nullable + public String hostname() { + return this.hostname; + } + + /** + * System's hostname. null in case of UnknownHostException + */ + @Nullable + public String getHostname() { + return hostname(); + } + /** * The service attributes of the node. */ @@ -184,6 +204,9 @@ public class NodeInfo extends NodeOperationResponse { @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); + if (in.readBoolean()) { + hostname = in.readUTF(); + } ImmutableMap.Builder builder = ImmutableMap.builder(); int size = in.readVInt(); for (int i = 0; i < size; i++) { @@ -214,6 +237,12 @@ public class NodeInfo extends NodeOperationResponse { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); + if (hostname == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeUTF(hostname); + } out.writeVInt(serviceAttributes.size()); for (Map.Entry entry : serviceAttributes.entrySet()) { out.writeUTF(entry.getKey()); diff --git a/src/main/java/org/elasticsearch/node/service/NodeService.java b/src/main/java/org/elasticsearch/node/service/NodeService.java index 86cc371799e..74955db9f47 100644 --- a/src/main/java/org/elasticsearch/node/service/NodeService.java +++ b/src/main/java/org/elasticsearch/node/service/NodeService.java @@ -19,6 +19,8 @@ package org.elasticsearch.node.service; +import java.net.InetAddress; + import com.google.common.collect.ImmutableMap; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; @@ -27,6 +29,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.network.NetworkUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.http.HttpServer; @@ -51,6 +54,9 @@ public class NodeService extends AbstractComponent { private volatile ImmutableMap serviceAttributes = ImmutableMap.of(); + @Nullable + private String hostname; + @Inject public NodeService(Settings settings, MonitorService monitorService, Discovery discovery, ClusterService clusterService, TransportService transportService, IndicesService indicesService) { super(settings); @@ -59,6 +65,10 @@ public class NodeService extends AbstractComponent { this.transportService = transportService; this.indicesService = indicesService; discovery.setNodeService(this); + InetAddress address = NetworkUtils.getLocalAddress(); + if (address != null) { + this.hostname = address.getHostName(); + } } public void setHttpServer(@Nullable HttpServer httpServer) { @@ -91,7 +101,7 @@ public class NodeService extends AbstractComponent { } public NodeInfo info() { - return new NodeInfo(clusterService.state().nodes().localNode(), serviceAttributes, settings, + return new NodeInfo(hostname, clusterService.state().nodes().localNode(), serviceAttributes, settings, monitorService.osService().info(), monitorService.processService().info(), monitorService.jvmService().info(), monitorService.networkService().info(), transportService.info(), httpServer == null ? null : httpServer.info()); diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java index 184e3ed9c6c..22d3a0ac3d8 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java @@ -73,6 +73,10 @@ public class RestNodesInfoAction extends BaseRestHandler { builder.field("name", nodeInfo.node().name(), XContentBuilder.FieldCaseConversion.NONE); builder.field("transport_address", nodeInfo.node().address().toString()); + if (nodeInfo.hostname() != null) { + builder.field("hostname", nodeInfo.hostname(), XContentBuilder.FieldCaseConversion.NONE); + } + for (Map.Entry nodeAttribute : nodeInfo.serviceAttributes().entrySet()) { builder.field(nodeAttribute.getKey(), nodeAttribute.getValue()); } From 16c4c8c726015c87f57ebb25e445c14d1d02e0c5 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 13 Dec 2011 19:11:36 +0200 Subject: [PATCH 032/270] mark hostname as nullable --- .../elasticsearch/action/admin/cluster/node/info/NodeInfo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java index d53a52bce46..07b4e3f6f5a 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java @@ -64,7 +64,7 @@ public class NodeInfo extends NodeOperationResponse { NodeInfo() { } - public NodeInfo(String hostname, DiscoveryNode node, ImmutableMap serviceAttributes, Settings settings, + public NodeInfo(@Nullable String hostname, DiscoveryNode node, ImmutableMap serviceAttributes, Settings settings, OsInfo os, ProcessInfo process, JvmInfo jvm, NetworkInfo network, TransportInfo transport, @Nullable HttpInfo http) { super(node); From 9ae402dde94d09c002272e14132f45167d3b9928 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 14 Dec 2011 15:07:36 +0200 Subject: [PATCH 033/270] Nested queries: getDocIdSet NullPointerException, closes #1536. --- .../index/query/NestedFilterParser.java | 155 ++++++++--------- .../index/query/NestedQueryParser.java | 161 +++++++++--------- 2 files changed, 162 insertions(+), 154 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/query/NestedFilterParser.java b/src/main/java/org/elasticsearch/index/query/NestedFilterParser.java index 712f5c017db..5aab30e927c 100644 --- a/src/main/java/org/elasticsearch/index/query/NestedFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/NestedFilterParser.java @@ -64,89 +64,92 @@ public class NestedFilterParser implements FilterParser { NestedQueryParser.LateBindingParentFilter usAsParentFilter = new NestedQueryParser.LateBindingParentFilter(); NestedQueryParser.parentFilterContext.set(usAsParentFilter); - String currentFieldName = null; - XContentParser.Token token; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - if ("query".equals(currentFieldName)) { - query = parseContext.parseInnerQuery(); - } else if ("filter".equals(currentFieldName)) { - filter = parseContext.parseInnerFilter(); - } - } else if (token.isValue()) { - if ("path".equals(currentFieldName)) { - path = parser.text(); - } else if ("boost".equals(currentFieldName)) { - boost = parser.floatValue(); - } else if ("_scope".equals(currentFieldName)) { - scope = parser.text(); - } else if ("_name".equals(currentFieldName)) { - filterName = parser.text(); - } else if ("_cache".equals(currentFieldName)) { - cache = parser.booleanValue(); - } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { - cacheKey = new CacheKeyFilter.Key(parser.text()); + try { + String currentFieldName = null; + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if ("query".equals(currentFieldName)) { + query = parseContext.parseInnerQuery(); + } else if ("filter".equals(currentFieldName)) { + filter = parseContext.parseInnerFilter(); + } + } else if (token.isValue()) { + if ("path".equals(currentFieldName)) { + path = parser.text(); + } else if ("boost".equals(currentFieldName)) { + boost = parser.floatValue(); + } else if ("_scope".equals(currentFieldName)) { + scope = parser.text(); + } else if ("_name".equals(currentFieldName)) { + filterName = parser.text(); + } else if ("_cache".equals(currentFieldName)) { + cache = parser.booleanValue(); + } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { + cacheKey = new CacheKeyFilter.Key(parser.text()); + } } } - } - if (query == null && filter == null) { - throw new QueryParsingException(parseContext.index(), "[nested] requires either 'query' or 'filter' field"); - } - if (path == null) { - throw new QueryParsingException(parseContext.index(), "[nested] requires 'path' field"); - } - - if (filter != null) { - query = new DeletionAwareConstantScoreQuery(filter); - } - - query.setBoost(boost); - - MapperService.SmartNameObjectMapper mapper = parseContext.smartObjectMapper(path); - if (mapper == null) { - throw new QueryParsingException(parseContext.index(), "[nested] failed to find nested object under path [" + path + "]"); - } - ObjectMapper objectMapper = mapper.mapper(); - if (objectMapper == null) { - throw new QueryParsingException(parseContext.index(), "[nested] failed to find nested object under path [" + path + "]"); - } - if (!objectMapper.nested().isNested()) { - throw new QueryParsingException(parseContext.index(), "[nested] nested object under path [" + path + "] is not of nested type"); - } - - Filter childFilter = parseContext.cacheFilter(objectMapper.nestedTypeFilter(), null); - usAsParentFilter.filter = childFilter; - // wrap the child query to only work on the nested path type - query = new FilteredQuery(query, childFilter); - - Filter parentFilter = currentParentFilterContext; - if (parentFilter == null) { - parentFilter = NonNestedDocsFilter.INSTANCE; - if (mapper.hasDocMapper()) { - // filter based on the type... - parentFilter = mapper.docMapper().typeFilter(); + if (query == null && filter == null) { + throw new QueryParsingException(parseContext.index(), "[nested] requires either 'query' or 'filter' field"); + } + if (path == null) { + throw new QueryParsingException(parseContext.index(), "[nested] requires 'path' field"); } - parentFilter = parseContext.cacheFilter(parentFilter, null); - } - // restore the thread local one... - NestedQueryParser.parentFilterContext.set(currentParentFilterContext); + if (filter != null) { + query = new DeletionAwareConstantScoreQuery(filter); + } - BlockJoinQuery joinQuery = new BlockJoinQuery(query, parentFilter, BlockJoinQuery.ScoreMode.None); + query.setBoost(boost); - if (scope != null) { - SearchContext.current().addNestedQuery(scope, joinQuery); - } + MapperService.SmartNameObjectMapper mapper = parseContext.smartObjectMapper(path); + if (mapper == null) { + throw new QueryParsingException(parseContext.index(), "[nested] failed to find nested object under path [" + path + "]"); + } + ObjectMapper objectMapper = mapper.mapper(); + if (objectMapper == null) { + throw new QueryParsingException(parseContext.index(), "[nested] failed to find nested object under path [" + path + "]"); + } + if (!objectMapper.nested().isNested()) { + throw new QueryParsingException(parseContext.index(), "[nested] nested object under path [" + path + "] is not of nested type"); + } - Filter joinFilter = new QueryWrapperFilter(joinQuery); - if (cache) { - joinFilter = parseContext.cacheFilter(joinFilter, cacheKey); + Filter childFilter = parseContext.cacheFilter(objectMapper.nestedTypeFilter(), null); + usAsParentFilter.filter = childFilter; + // wrap the child query to only work on the nested path type + query = new FilteredQuery(query, childFilter); + + Filter parentFilter = currentParentFilterContext; + if (parentFilter == null) { + parentFilter = NonNestedDocsFilter.INSTANCE; + if (mapper.hasDocMapper()) { + // filter based on the type... + parentFilter = mapper.docMapper().typeFilter(); + } + parentFilter = parseContext.cacheFilter(parentFilter, null); + } + + BlockJoinQuery joinQuery = new BlockJoinQuery(query, parentFilter, BlockJoinQuery.ScoreMode.None); + + if (scope != null) { + SearchContext.current().addNestedQuery(scope, joinQuery); + } + + Filter joinFilter = new QueryWrapperFilter(joinQuery); + if (cache) { + joinFilter = parseContext.cacheFilter(joinFilter, cacheKey); + } + if (filterName != null) { + parseContext.addNamedFilter(filterName, joinFilter); + } + return joinFilter; + + } finally { + // restore the thread local one... + NestedQueryParser.parentFilterContext.set(currentParentFilterContext); } - if (filterName != null) { - parseContext.addNamedFilter(filterName, joinFilter); - } - return joinFilter; } } diff --git a/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java b/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java index 8b515b6962d..b5d2d0c431b 100644 --- a/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java @@ -62,89 +62,94 @@ public class NestedQueryParser implements QueryParser { LateBindingParentFilter usAsParentFilter = new LateBindingParentFilter(); parentFilterContext.set(usAsParentFilter); - String currentFieldName = null; - XContentParser.Token token; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - if ("query".equals(currentFieldName)) { - query = parseContext.parseInnerQuery(); - } else if ("filter".equals(currentFieldName)) { - filter = parseContext.parseInnerFilter(); - } - } else if (token.isValue()) { - if ("path".equals(currentFieldName)) { - path = parser.text(); - } else if ("boost".equals(currentFieldName)) { - boost = parser.floatValue(); - } else if ("_scope".equals(currentFieldName)) { - scope = parser.text(); - } else if ("score_mode".equals(currentFieldName) || "scoreMode".equals(currentFieldName)) { - String sScoreMode = parser.text(); - if ("avg".equals(sScoreMode)) { - scoreMode = BlockJoinQuery.ScoreMode.Avg; - } else if ("max".equals(sScoreMode)) { - scoreMode = BlockJoinQuery.ScoreMode.Max; - } else if ("total".equals(sScoreMode)) { - scoreMode = BlockJoinQuery.ScoreMode.Total; - } else if ("none".equals(sScoreMode)) { - scoreMode = BlockJoinQuery.ScoreMode.None; - } else { - throw new QueryParsingException(parseContext.index(), "illegal score_mode for nested query [" + sScoreMode + "]"); + try { + String currentFieldName = null; + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if ("query".equals(currentFieldName)) { + query = parseContext.parseInnerQuery(); + } else if ("filter".equals(currentFieldName)) { + filter = parseContext.parseInnerFilter(); + } + } else if (token.isValue()) { + if ("path".equals(currentFieldName)) { + path = parser.text(); + } else if ("boost".equals(currentFieldName)) { + boost = parser.floatValue(); + } else if ("_scope".equals(currentFieldName)) { + scope = parser.text(); + } else if ("score_mode".equals(currentFieldName) || "scoreMode".equals(currentFieldName)) { + String sScoreMode = parser.text(); + if ("avg".equals(sScoreMode)) { + scoreMode = BlockJoinQuery.ScoreMode.Avg; + } else if ("max".equals(sScoreMode)) { + scoreMode = BlockJoinQuery.ScoreMode.Max; + } else if ("total".equals(sScoreMode)) { + scoreMode = BlockJoinQuery.ScoreMode.Total; + } else if ("none".equals(sScoreMode)) { + scoreMode = BlockJoinQuery.ScoreMode.None; + } else { + throw new QueryParsingException(parseContext.index(), "illegal score_mode for nested query [" + sScoreMode + "]"); + } } } } - } - if (query == null && filter == null) { - throw new QueryParsingException(parseContext.index(), "[nested] requires either 'query' or 'filter' field"); - } - if (path == null) { - throw new QueryParsingException(parseContext.index(), "[nested] requires 'path' field"); - } - - if (filter != null) { - query = new DeletionAwareConstantScoreQuery(filter); - } - - MapperService.SmartNameObjectMapper mapper = parseContext.smartObjectMapper(path); - if (mapper == null) { - throw new QueryParsingException(parseContext.index(), "[nested] failed to find nested object under path [" + path + "]"); - } - ObjectMapper objectMapper = mapper.mapper(); - if (objectMapper == null) { - throw new QueryParsingException(parseContext.index(), "[nested] failed to find nested object under path [" + path + "]"); - } - if (!objectMapper.nested().isNested()) { - throw new QueryParsingException(parseContext.index(), "[nested] nested object under path [" + path + "] is not of nested type"); - } - - Filter childFilter = parseContext.cacheFilter(objectMapper.nestedTypeFilter(), null); - usAsParentFilter.filter = childFilter; - // wrap the child query to only work on the nested path type - query = new FilteredQuery(query, childFilter); - - Filter parentFilter = currentParentFilterContext; - if (parentFilter == null) { - parentFilter = NonNestedDocsFilter.INSTANCE; - if (mapper.hasDocMapper()) { - // filter based on the type... - parentFilter = mapper.docMapper().typeFilter(); + if (query == null && filter == null) { + throw new QueryParsingException(parseContext.index(), "[nested] requires either 'query' or 'filter' field"); } - parentFilter = parseContext.cacheFilter(parentFilter, null); + if (path == null) { + throw new QueryParsingException(parseContext.index(), "[nested] requires 'path' field"); + } + + if (filter != null) { + query = new DeletionAwareConstantScoreQuery(filter); + } + + MapperService.SmartNameObjectMapper mapper = parseContext.smartObjectMapper(path); + if (mapper == null) { + throw new QueryParsingException(parseContext.index(), "[nested] failed to find nested object under path [" + path + "]"); + } + ObjectMapper objectMapper = mapper.mapper(); + if (objectMapper == null) { + throw new QueryParsingException(parseContext.index(), "[nested] failed to find nested object under path [" + path + "]"); + } + if (!objectMapper.nested().isNested()) { + throw new QueryParsingException(parseContext.index(), "[nested] nested object under path [" + path + "] is not of nested type"); + } + + Filter childFilter = parseContext.cacheFilter(objectMapper.nestedTypeFilter(), null); + usAsParentFilter.filter = childFilter; + if (usAsParentFilter.filter == null) { + System.out.println("HELLO"); + } + // wrap the child query to only work on the nested path type + query = new FilteredQuery(query, childFilter); + + Filter parentFilter = currentParentFilterContext; + if (parentFilter == null) { + parentFilter = NonNestedDocsFilter.INSTANCE; + if (mapper.hasDocMapper()) { + // filter based on the type... + parentFilter = mapper.docMapper().typeFilter(); + } + parentFilter = parseContext.cacheFilter(parentFilter, null); + } + + BlockJoinQuery joinQuery = new BlockJoinQuery(query, parentFilter, scoreMode); + joinQuery.setBoost(boost); + + if (scope != null) { + SearchContext.current().addNestedQuery(scope, joinQuery); + } + + return joinQuery; + } finally { + // restore the thread local one... + parentFilterContext.set(currentParentFilterContext); } - - // restore the thread local one... - parentFilterContext.set(currentParentFilterContext); - - BlockJoinQuery joinQuery = new BlockJoinQuery(query, parentFilter, scoreMode); - joinQuery.setBoost(boost); - - if (scope != null) { - SearchContext.current().addNestedQuery(scope, joinQuery); - } - - return joinQuery; } static ThreadLocal parentFilterContext = new ThreadLocal(); From f05fa91cb2beed59f86214afe51b0335d258eff8 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 14 Dec 2011 16:28:29 +0200 Subject: [PATCH 034/270] if we loaded source, use it when needing it in other fetch phases like extracting specific fields instead of loading it again --- .../index/get/ShardGetService.java | 2 +- .../search/fetch/FetchPhase.java | 3 +++ .../search/lookup/SourceLookup.java | 21 +++++++++++++------ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 8af217bf3ba..43dfc6a2c57 100644 --- a/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -293,7 +293,7 @@ public class ShardGetService extends AbstractIndexShardComponent { } else { if (searchLookup == null) { searchLookup = new SearchLookup(mapperService, indexCache.fieldData()); - searchLookup.source().setNextSource(SourceLookup.sourceAsMap(source.source.bytes(), source.source.offset(), source.source.length())); + searchLookup.source().setNextSource(source.source.bytes(), source.source.offset(), source.source.length()); } FieldMapper x = docMapper.mappers().smartNameFieldMapper(field); diff --git a/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index ca0c1af3e08..79169cd6c26 100644 --- a/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -184,6 +184,9 @@ public class FetchPhase implements SearchPhase { // go over and extract fields that are not mapped / stored context.lookup().setNextReader(subReader); context.lookup().setNextDocId(subDoc); + if (source != null) { + context.lookup().source().setNextSource(source, 0, source.length); + } if (extractFieldNames != null) { for (String extractFieldName : extractFieldNames) { Object value = context.lookup().source().extractValue(extractFieldName); diff --git a/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index 7047c06f432..655c5c74b45 100644 --- a/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -25,7 +25,6 @@ import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.IndexReader; import org.elasticsearch.ElasticSearchParseException; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.mapper.internal.SourceFieldMapper; import org.elasticsearch.index.mapper.internal.SourceFieldSelector; @@ -45,6 +44,9 @@ public class SourceLookup implements Map { private int docId = -1; + private byte[] sourceAsBytes; + private int sourceAsBytesOffset; + private int sourceAsBytesLength; private Map source; public Map source() { @@ -55,7 +57,10 @@ public class SourceLookup implements Map { if (source != null) { return source; } - XContentParser parser = null; + if (sourceAsBytes != null) { + source = sourceAsMap(sourceAsBytes, sourceAsBytesOffset, sourceAsBytesLength); + return source; + } try { Document doc = reader.document(docId, SourceFieldSelector.INSTANCE); Fieldable sourceField = doc.getFieldable(SourceFieldMapper.NAME); @@ -66,10 +71,6 @@ public class SourceLookup implements Map { } } catch (Exception e) { throw new ElasticSearchParseException("failed to parse / load source", e); - } finally { - if (parser != null) { - parser.close(); - } } return this.source; } @@ -84,6 +85,7 @@ public class SourceLookup implements Map { } this.reader = reader; this.source = null; + this.sourceAsBytes = null; this.docId = -1; } @@ -92,9 +94,16 @@ public class SourceLookup implements Map { return; } this.docId = docId; + this.sourceAsBytes = null; this.source = null; } + public void setNextSource(byte[] source, int offset, int length) { + this.sourceAsBytes = source; + this.sourceAsBytesOffset = offset; + this.sourceAsBytesLength = length; + } + public void setNextSource(Map source) { this.source = source; } From 922833cdc4794a9a46bcbf5df5735308f6e643c1 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 14 Dec 2011 21:18:14 +0200 Subject: [PATCH 035/270] source not returned when * specified in fields list, closes #1541. --- .../search/fetch/FetchPhase.java | 50 +++++++++++++++---- .../search/fields/SearchFieldsTests.java | 11 +++- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 79169cd6c26..620428ef7a3 100644 --- a/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -24,6 +24,7 @@ import com.google.common.collect.Lists; import org.apache.lucene.document.Document; import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.IndexReader; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.document.ResetFieldSelector; import org.elasticsearch.index.Index; @@ -87,22 +88,34 @@ public class FetchPhase implements SearchPhase { public void execute(SearchContext context) { ResetFieldSelector fieldSelector; List extractFieldNames = null; + boolean sourceRequested = false; if (context.hasScriptFields() && !context.hasFieldNames()) { // we ask for script fields, and no field names, don't load the source fieldSelector = UidFieldSelector.INSTANCE; + sourceRequested = false; } else if (!context.hasFieldNames()) { fieldSelector = new UidAndSourceFieldSelector(); + sourceRequested = true; } else if (context.fieldNames().isEmpty()) { fieldSelector = UidFieldSelector.INSTANCE; - } else if (context.fieldNames().get(0).equals("*")) { - // asked for all stored fields, just return null so all of them will be loaded - // don't load the source field in this case, makes little sense to get it with all stored fields - fieldSelector = AllButSourceFieldSelector.INSTANCE; + sourceRequested = false; } else { - FieldMappersFieldSelector fieldSelectorMapper = new FieldMappersFieldSelector(); + boolean loadAllStored = false; + FieldMappersFieldSelector fieldSelectorMapper = null; for (String fieldName : context.fieldNames()) { + if (fieldName.equals("*")) { + loadAllStored = true; + continue; + } + if (fieldName.equals(SourceFieldMapper.NAME)) { + sourceRequested = true; + continue; + } FieldMappers x = context.smartNameFieldMappers(fieldName); if (x != null && x.mapper().stored()) { + if (fieldSelectorMapper == null) { + fieldSelectorMapper = new FieldMappersFieldSelector(); + } fieldSelectorMapper.add(x); } else { if (extractFieldNames == null) { @@ -111,8 +124,25 @@ public class FetchPhase implements SearchPhase { extractFieldNames.add(fieldName); } } - fieldSelectorMapper.add(UidFieldMapper.NAME); - fieldSelector = fieldSelectorMapper; + + if (loadAllStored) { + if (sourceRequested || extractFieldNames != null) { + fieldSelector = null; // load everything, including _source + } else { + fieldSelector = AllButSourceFieldSelector.INSTANCE; + } + } else if (fieldSelectorMapper != null) { + // we are asking specific stored fields, just add the UID and be done + fieldSelectorMapper.add(UidFieldMapper.NAME); + if (extractFieldNames != null) { + fieldSelectorMapper.add(SourceFieldMapper.NAME); + } + fieldSelector = fieldSelectorMapper; + } else if (extractFieldNames != null) { + fieldSelector = new UidAndSourceFieldSelector(); + } else { + fieldSelector = UidFieldSelector.INSTANCE; + } } InternalSearchHit[] hits = new InternalSearchHit[context.docIdsToLoadSize()]; @@ -131,7 +161,7 @@ public class FetchPhase implements SearchPhase { // get the version - InternalSearchHit searchHit = new InternalSearchHit(docId, uid.id(), uid.type(), source, null); + InternalSearchHit searchHit = new InternalSearchHit(docId, uid.id(), uid.type(), sourceRequested ? source : null, null); hits[index] = searchHit; for (Object oField : doc.getFields()) { @@ -245,9 +275,9 @@ public class FetchPhase implements SearchPhase { throw new FetchPhaseExecutionException(context, "Failed to load uid from the index, missing internal _uid field, current fields in the doc [" + fieldNames + "]"); } - private Document loadDocument(SearchContext context, ResetFieldSelector fieldSelector, int docId) { + private Document loadDocument(SearchContext context, @Nullable ResetFieldSelector fieldSelector, int docId) { try { - fieldSelector.reset(); + if (fieldSelector != null) fieldSelector.reset(); return context.searcher().doc(docId, fieldSelector); } catch (IOException e) { throw new FetchPhaseExecutionException(context, "Failed to fetch doc id [" + docId + "]", e); diff --git a/src/test/java/org/elasticsearch/test/integration/search/fields/SearchFieldsTests.java b/src/test/java/org/elasticsearch/test/integration/search/fields/SearchFieldsTests.java index 948ab4694f7..56abdb0c181 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/fields/SearchFieldsTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/fields/SearchFieldsTests.java @@ -30,7 +30,7 @@ import org.testng.annotations.Test; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.*; /** * @@ -99,6 +99,15 @@ public class SearchFieldsTests extends AbstractNodesTests { searchResponse = client.prepareSearch().setQuery(matchAllQuery()).addField("*").execute().actionGet(); assertThat(searchResponse.hits().getTotalHits(), equalTo(1l)); assertThat(searchResponse.hits().hits().length, equalTo(1)); + assertThat(searchResponse.hits().getAt(0).source(), nullValue()); + assertThat(searchResponse.hits().getAt(0).fields().size(), equalTo(2)); + assertThat(searchResponse.hits().getAt(0).fields().get("field1").value().toString(), equalTo("value1")); + assertThat(searchResponse.hits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); + + searchResponse = client.prepareSearch().setQuery(matchAllQuery()).addField("*").addField("_source").execute().actionGet(); + assertThat(searchResponse.hits().getTotalHits(), equalTo(1l)); + assertThat(searchResponse.hits().hits().length, equalTo(1)); + assertThat(searchResponse.hits().getAt(0).source(), notNullValue()); assertThat(searchResponse.hits().getAt(0).fields().size(), equalTo(2)); assertThat(searchResponse.hits().getAt(0).fields().get("field1").value().toString(), equalTo("value1")); assertThat(searchResponse.hits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); From f745b80edb3890b7e293aaa9b95496d099f1d6e0 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 15 Dec 2011 20:49:38 +0200 Subject: [PATCH 036/270] timestamp term match broken in 0.18.5, closes #1543. --- .../index/mapper/core/DateFieldMapper.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index 25165f8ff2e..c74212476a6 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.NumericRangeFilter; import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.Strings; import org.elasticsearch.common.joda.FormatDateTimeFormatter; @@ -38,6 +39,7 @@ import org.elasticsearch.index.analysis.NumericDateAnalyzer; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.search.NumericRangeFieldDataFilter; import org.joda.time.DateTimeZone; import org.joda.time.MutableDateTime; @@ -227,6 +229,13 @@ public class DateFieldMapper extends NumberFieldMapper { true, true); } + @Override + public Query fieldQuery(String value, @Nullable QueryParseContext context) { + long lValue = parseStringValue(value); + return NumericRangeQuery.newLongRange(names.indexName(), precisionStep, + lValue, lValue, true, true); + } + @Override public Query rangeQuery(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { return NumericRangeQuery.newLongRange(names.indexName(), precisionStep, @@ -235,6 +244,13 @@ public class DateFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override + public Filter fieldFilter(String value, @Nullable QueryParseContext context) { + long lValue = parseStringValue(value); + return NumericRangeFilter.newLongRange(names.indexName(), precisionStep, + lValue, lValue, true, true); + } + @Override public Filter rangeFilter(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { return NumericRangeFilter.newLongRange(names.indexName(), precisionStep, From 92c162d5345a7b3657651c12bca77384f70bb437 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 15 Dec 2011 21:24:28 +0200 Subject: [PATCH 037/270] optimize field query/filter for numerics to only parse the string representation once --- .../index/mapper/core/ByteFieldMapper.java | 15 +++++++++++++++ .../index/mapper/core/DoubleFieldMapper.java | 16 ++++++++++++++++ .../index/mapper/core/FloatFieldMapper.java | 16 ++++++++++++++++ .../index/mapper/core/IntegerFieldMapper.java | 16 ++++++++++++++++ .../index/mapper/core/LongFieldMapper.java | 16 ++++++++++++++++ .../index/mapper/core/ShortFieldMapper.java | 16 ++++++++++++++++ 6 files changed, 95 insertions(+) diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java index 29eaea76cba..6601558fd4d 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.NumericRangeFilter; import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -35,6 +36,7 @@ import org.elasticsearch.index.analysis.NumericIntegerAnalyzer; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.search.NumericRangeFieldDataFilter; import java.io.IOException; @@ -157,6 +159,12 @@ public class ByteFieldMapper extends NumberFieldMapper { true, true); } + @Override + public Query fieldQuery(String value, @Nullable QueryParseContext context) { + int iValue = Integer.parseInt(value); + return NumericRangeQuery.newIntRange(names.indexName(), precisionStep, + iValue, iValue, true, true); + } @Override public Query rangeQuery(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { @@ -166,6 +174,13 @@ public class ByteFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override + public Filter fieldFilter(String value, @Nullable QueryParseContext context) { + int iValue = Integer.parseInt(value); + return NumericRangeFilter.newIntRange(names.indexName(), precisionStep, + iValue, iValue, true, true); + } + @Override public Filter rangeFilter(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { return NumericRangeFilter.newIntRange(names.indexName(), precisionStep, diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java index 80103d406f3..eab5ae5fb3e 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.NumericRangeFilter; import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -35,6 +36,7 @@ import org.elasticsearch.index.analysis.NumericDoubleAnalyzer; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.search.NumericRangeFieldDataFilter; import java.io.IOException; @@ -154,6 +156,13 @@ public class DoubleFieldMapper extends NumberFieldMapper { true, true); } + @Override + public Query fieldQuery(String value, @Nullable QueryParseContext context) { + double dValue = Double.parseDouble(value); + return NumericRangeQuery.newDoubleRange(names.indexName(), precisionStep, + dValue, dValue, true, true); + } + @Override public Query rangeQuery(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { return NumericRangeQuery.newDoubleRange(names.indexName(), precisionStep, @@ -162,6 +171,13 @@ public class DoubleFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override + public Filter fieldFilter(String value, @Nullable QueryParseContext context) { + double dValue = Double.parseDouble(value); + return NumericRangeFilter.newDoubleRange(names.indexName(), precisionStep, + dValue, dValue, true, true); + } + @Override public Filter rangeFilter(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { return NumericRangeFilter.newDoubleRange(names.indexName(), precisionStep, diff --git a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java index e665744299f..0e3daec618c 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.NumericRangeFilter; import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -36,6 +37,7 @@ import org.elasticsearch.index.analysis.NumericFloatAnalyzer; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.search.NumericRangeFieldDataFilter; import java.io.IOException; @@ -153,6 +155,13 @@ public class FloatFieldMapper extends NumberFieldMapper { true, true); } + @Override + public Query fieldQuery(String value, @Nullable QueryParseContext context) { + float fValue = Float.parseFloat(value); + return NumericRangeQuery.newFloatRange(names.indexName(), precisionStep, + fValue, fValue, true, true); + } + @Override public Query rangeQuery(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { return NumericRangeQuery.newFloatRange(names.indexName(), precisionStep, @@ -161,6 +170,13 @@ public class FloatFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override + public Filter fieldFilter(String value, @Nullable QueryParseContext context) { + float fValue = Float.parseFloat(value); + return NumericRangeFilter.newFloatRange(names.indexName(), precisionStep, + fValue, fValue, true, true); + } + @Override public Filter rangeFilter(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { return NumericRangeFilter.newFloatRange(names.indexName(), precisionStep, diff --git a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java index 50b1d3edc63..6f63d089380 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.NumericRangeFilter; import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -36,6 +37,7 @@ import org.elasticsearch.index.analysis.NumericIntegerAnalyzer; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.search.NumericRangeFieldDataFilter; import java.io.IOException; @@ -158,6 +160,13 @@ public class IntegerFieldMapper extends NumberFieldMapper { true, true); } + @Override + public Query fieldQuery(String value, @Nullable QueryParseContext context) { + int iValue = Integer.parseInt(value); + return NumericRangeQuery.newIntRange(names.indexName(), precisionStep, + iValue, iValue, true, true); + } + @Override public Query rangeQuery(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { return NumericRangeQuery.newIntRange(names.indexName(), precisionStep, @@ -166,6 +175,13 @@ public class IntegerFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override + public Filter fieldFilter(String value, @Nullable QueryParseContext context) { + int iValue = Integer.parseInt(value); + return NumericRangeFilter.newIntRange(names.indexName(), precisionStep, + iValue, iValue, true, true); + } + @Override public Filter rangeFilter(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { return NumericRangeFilter.newIntRange(names.indexName(), precisionStep, diff --git a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java index b970500c030..22e83231c65 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.NumericRangeFilter; import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -36,6 +37,7 @@ import org.elasticsearch.index.analysis.NumericLongAnalyzer; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.search.NumericRangeFieldDataFilter; import java.io.IOException; @@ -158,6 +160,13 @@ public class LongFieldMapper extends NumberFieldMapper { true, true); } + @Override + public Query fieldQuery(String value, @Nullable QueryParseContext context) { + long iValue = Long.parseLong(value); + return NumericRangeQuery.newLongRange(names.indexName(), precisionStep, + iValue, iValue, true, true); + } + @Override public Query rangeQuery(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { return NumericRangeQuery.newLongRange(names.indexName(), precisionStep, @@ -166,6 +175,13 @@ public class LongFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override + public Filter fieldFilter(String value, @Nullable QueryParseContext context) { + long iValue = Long.parseLong(value); + return NumericRangeFilter.newLongRange(names.indexName(), precisionStep, + iValue, iValue, true, true); + } + @Override public Filter rangeFilter(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { return NumericRangeFilter.newLongRange(names.indexName(), precisionStep, diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java index 28b5a66d735..0202b715a1a 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.NumericRangeFilter; import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -36,6 +37,7 @@ import org.elasticsearch.index.analysis.NumericIntegerAnalyzer; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.search.NumericRangeFieldDataFilter; import java.io.IOException; @@ -158,6 +160,13 @@ public class ShortFieldMapper extends NumberFieldMapper { true, true); } + @Override + public Query fieldQuery(String value, @Nullable QueryParseContext context) { + int iValue = Integer.parseInt(value); + return NumericRangeQuery.newIntRange(names.indexName(), precisionStep, + iValue, iValue, true, true); + } + @Override public Query rangeQuery(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { return NumericRangeQuery.newIntRange(names.indexName(), precisionStep, @@ -166,6 +175,13 @@ public class ShortFieldMapper extends NumberFieldMapper { includeLower, includeUpper); } + @Override + public Filter fieldFilter(String value, @Nullable QueryParseContext context) { + int iValue = Integer.parseInt(value); + return NumericRangeFilter.newIntRange(names.indexName(), precisionStep, + iValue, iValue, true, true); + } + @Override public Filter rangeFilter(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { return NumericRangeFilter.newIntRange(names.indexName(), precisionStep, From 1a26ba796803bd57a312f7dcf49e965dea4b746d Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 15 Dec 2011 22:36:44 +0200 Subject: [PATCH 038/270] improve guessing if upper date range check works or is it a numeric --- .../index/mapper/core/DateFieldMapper.java | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index c74212476a6..2dd21575bca 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -402,7 +402,7 @@ public class DateFieldMapper extends NumberFieldMapper { long time = Long.parseLong(value); return timeUnit.toMillis(time); } catch (NumberFormatException e1) { - throw new MapperParsingException("failed to parse date field, tried both date format [" + dateTimeFormatter.format() + "], and timestamp number", e); + throw new MapperParsingException("failed to parse date field [" + value + "], tried both date format [" + dateTimeFormatter.format() + "], and timestamp number", e); } } } @@ -413,14 +413,28 @@ public class DateFieldMapper extends NumberFieldMapper { } try { MutableDateTime dateTime = new MutableDateTime(3000, 12, 31, 23, 59, 59, 999, DateTimeZone.UTC); - dateTimeFormatter.parser().parseInto(dateTime, value, 0); + int location = dateTimeFormatter.parser().parseInto(dateTime, value, 0); + // if we parsed all the string value, we are good + if (location == value.length()) { + return dateTime.getMillis(); + } + // if we did not manage to parse, or the year is really high year which is unreasonable + // see if its a number + if (location <= 0 || dateTime.getYear() > 5000) { + try { + long time = Long.parseLong(value); + return timeUnit.toMillis(time); + } catch (NumberFormatException e1) { + throw new MapperParsingException("failed to parse date field [" + value + "], tried both date format [" + dateTimeFormatter.format() + "], and timestamp number"); + } + } return dateTime.getMillis(); } catch (RuntimeException e) { try { long time = Long.parseLong(value); return timeUnit.toMillis(time); } catch (NumberFormatException e1) { - throw new MapperParsingException("failed to parse date field, tried both date format [" + dateTimeFormatter.format() + "], and timestamp number", e); + throw new MapperParsingException("failed to parse date field [" + value + "], tried both date format [" + dateTimeFormatter.format() + "], and timestamp number", e); } } } From 367a60870787c5374be008e4841e89c16a0fe2cc Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 16 Dec 2011 15:36:22 +0200 Subject: [PATCH 039/270] remove jline from distribution to simplify it (no longer painting log levels though...) --- pom.xml | 10 +- src/main/assemblies/common-bin.xml | 1 - .../elasticsearch/bootstrap/Bootstrap.java | 20 +- .../org/elasticsearch/common/jline/ANSI.java | 381 ------------------ .../logging/log4j/JLinePatternLayout.java | 101 ----- .../common/logging/log4j/LogConfigurator.java | 2 +- 6 files changed, 3 insertions(+), 512 deletions(-) delete mode 100644 src/main/java/org/elasticsearch/common/jline/ANSI.java delete mode 100644 src/main/java/org/elasticsearch/common/logging/log4j/JLinePatternLayout.java diff --git a/pom.xml b/pom.xml index 1d8b395367b..2aa96354f6d 100644 --- a/pom.xml +++ b/pom.xml @@ -196,14 +196,6 @@ true - - jline - jline - 1.0 - compile - true - - sigar sigar @@ -493,7 +485,7 @@ ${project.build.directory}/lib - lucene*, log4j*, jline*, jna* + lucene*, log4j*, jna* directory perm diff --git a/src/main/assemblies/common-bin.xml b/src/main/assemblies/common-bin.xml index c6df88a26ac..5ef827435c8 100644 --- a/src/main/assemblies/common-bin.xml +++ b/src/main/assemblies/common-bin.xml @@ -6,7 +6,6 @@ org.apache.lucene:lucene* log4j:log4j - jline:jline net.java.dev.jna:jna diff --git a/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index efd12ba3490..4d7655de8eb 100644 --- a/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -26,12 +26,10 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.CreationException; import org.elasticsearch.common.inject.spi.Message; import org.elasticsearch.common.io.FileSystemUtils; -import org.elasticsearch.common.jline.ANSI; import org.elasticsearch.common.jna.Natives; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.logging.log4j.LogConfigurator; -import org.elasticsearch.common.os.OsUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.monitor.jvm.JvmInfo; @@ -44,13 +42,10 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import static com.google.common.collect.Sets.newHashSet; -import static jline.ANSIBuffer.ANSICodes.attrib; import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_SETTINGS; /** * A main entry point when starting from the command line. - * - * */ public class Bootstrap { @@ -149,11 +144,6 @@ public class Bootstrap { bootstrap = new Bootstrap(); String pidFile = System.getProperty("es-pidfile"); - // enable jline by default when running form "main" (and not on windows) - if (System.getProperty("jline.enabled") == null && !OsUtils.WINDOWS) { - System.setProperty("jline.enabled", "true"); - } - boolean foreground = System.getProperty("es-foreground") != null; // handle the wrapper system property, if its a service, don't run as a service if (System.getProperty("wrapper.service", "XXX").equalsIgnoreCase("true")) { @@ -244,15 +234,7 @@ public class Bootstrap { private static String buildErrorMessage(String stage, Throwable e) { StringBuilder errorMessage = new StringBuilder("{").append(Version.CURRENT).append("}: "); - try { - if (ANSI.isEnabled()) { - errorMessage.append(attrib(ANSI.Code.FG_RED)).append(stage).append(" Failed ...").append(attrib(ANSI.Code.OFF)).append("\n"); - } else { - errorMessage.append(stage).append(" Failed ...\n"); - } - } catch (Throwable t) { - errorMessage.append(stage).append(" Failed ...\n"); - } + errorMessage.append(stage).append(" Failed ...\n"); if (e instanceof CreationException) { CreationException createException = (CreationException) e; Set seenMessages = newHashSet(); diff --git a/src/main/java/org/elasticsearch/common/jline/ANSI.java b/src/main/java/org/elasticsearch/common/jline/ANSI.java deleted file mode 100644 index 87b5db286fd..00000000000 --- a/src/main/java/org/elasticsearch/common/jline/ANSI.java +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.jline; - -import jline.ANSIBuffer; -import jline.Terminal; - -import java.io.OutputStream; -import java.io.PrintWriter; -import java.io.Writer; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.util.HashMap; -import java.util.Map; - -/** - * - */ -public class ANSI { - // - // Detection/Enabled Muck - // - - /** - * Tries to detect if the current system supports ANSI. - */ - private static boolean detect() { - if (System.getProperty("jline.enabled", "false").equalsIgnoreCase("false")) { - return false; - } - boolean enabled = Terminal.getTerminal().isANSISupported(); - - if (!enabled) { - String force = System.getProperty(ANSI.class.getName() + ".force", "false"); - enabled = Boolean.valueOf(force).booleanValue(); - } - - return enabled; - } - - public static boolean isDetected() { - return detect(); - } - - private static Boolean enabled; - - public static void setEnabled(final boolean flag) { - enabled = Boolean.valueOf(flag); - } - - public static boolean isEnabled() { - if (enabled == null) { - enabled = Boolean.valueOf(isDetected()); - } - - return enabled.booleanValue(); - } - - // - // Code - // - - public static class Code { - // - // NOTE: Some fields duplicated from jline.ANSIBuffer.ANSICodes to change access modifiers - // - - public static final int OFF = 0; - public static final int BOLD = 1; - public static final int UNDERSCORE = 4; - public static final int BLINK = 5; - public static final int REVERSE = 7; - public static final int CONCEALED = 8; - - public static final int FG_BLACK = 30; - public static final int FG_RED = 31; - public static final int FG_GREEN = 32; - public static final int FG_YELLOW = 33; - public static final int FG_BLUE = 34; - public static final int FG_MAGENTA = 35; - public static final int FG_CYAN = 36; - public static final int FG_WHITE = 37; - - public static final int BLACK = FG_BLACK; - public static final int RED = FG_RED; - public static final int GREEN = FG_GREEN; - public static final int YELLOW = FG_YELLOW; - public static final int BLUE = FG_BLUE; - public static final int MAGENTA = FG_MAGENTA; - public static final int CYAN = FG_CYAN; - public static final int WHITE = FG_WHITE; - - public static final int BG_BLACK = 40; - public static final int BG_RED = 41; - public static final int BG_GREEN = 42; - public static final int BG_YELLOW = 43; - public static final int BG_BLUE = 44; - public static final int BG_MAGENTA = 45; - public static final int BG_CYAN = 46; - public static final int BG_WHITE = 47; - - /** - * A map of code names to values. - */ - private static final Map NAMES_TO_CODES; - - /** - * A map of codes to name. - */ - private static final Map CODES_TO_NAMES; - - static { - Field[] fields = Code.class.getDeclaredFields(); - Map names = new HashMap(fields.length); - Map codes = new HashMap(fields.length); - - try { - for (int i = 0; i < fields.length; i++) { - // Skip anything non-public, all public fields are codes - int mods = fields[i].getModifiers(); - if (!Modifier.isPublic(mods)) { - continue; - } - - String name = fields[i].getName(); - Number code = (Number) fields[i].get(Code.class); - - names.put(name, code); - codes.put(code, name); - } - } catch (IllegalAccessException e) { - // This should never happen - throw new Error(e); - } - - NAMES_TO_CODES = names; - CODES_TO_NAMES = codes; - } - - /** - * Returns the ANSI code for the given symbolic name. Supported symbolic names are all defined as - * fields in {@link ANSI.Code} where the case is not significant. - */ - public static int forName(final String name) throws IllegalArgumentException { - assert name != null; - - // All names in the map are upper-case - String tmp = name.toUpperCase(); - Number code = (Number) NAMES_TO_CODES.get(tmp); - - if (code == null) { - throw new IllegalArgumentException("Invalid ANSI code name: " + name); - } - - return code.intValue(); - } - - /** - * Returns the symbolic name for the given ANSI code. - */ - public static String name(final int code) throws IllegalArgumentException { - String name = (String) CODES_TO_NAMES.get(Integer.valueOf(code)); - - if (name == null) { - throw new IllegalArgumentException("Invalid ANSI code: " + code); - } - - return name; - } - } - - // - // Buffer - // - - public static class Buffer { - private final StringBuffer buff = new StringBuffer(); - - public final boolean autoClear = true; - - public String toString() { - try { - return buff.toString(); - } finally { - if (autoClear) clear(); - } - } - - public void clear() { - buff.setLength(0); - } - - public int size() { - return buff.length(); - } - - public Buffer append(final String text) { - buff.append(text); - - return this; - } - - public Buffer append(final Object obj) { - return append(String.valueOf(obj)); - } - - public Buffer attrib(final int code) { - if (isEnabled()) { - buff.append(ANSIBuffer.ANSICodes.attrib(code)); - } - - return this; - } - - public Buffer attrib(final String text, final int code) { - assert text != null; - - if (isEnabled()) { - buff.append(ANSIBuffer.ANSICodes.attrib(code)).append(text).append(ANSIBuffer.ANSICodes.attrib(Code.OFF)); - } else { - buff.append(text); - } - - return this; - } - - public Buffer attrib(final String text, final String codeName) { - return attrib(text, Code.forName(codeName)); - } - } - - // - // Renderer - // - - public static class Renderer { - public static final String BEGIN_TOKEN = "@|"; - - private static final int BEGIN_TOKEN_SIZE = BEGIN_TOKEN.length(); - - public static final String END_TOKEN = "|"; - - private static final int END_TOKEN_SIZE = END_TOKEN.length(); - - public static final String CODE_TEXT_SEPARATOR = " "; - - public static final String CODE_LIST_SEPARATOR = ","; - - private final Buffer buff = new Buffer(); - - public String render(final String input) throws RenderException { - assert input != null; - - // current, prefix and suffix positions - int c = 0, p, s; - - while (c < input.length()) { - p = input.indexOf(BEGIN_TOKEN, c); - if (p < 0) { - break; - } - - s = input.indexOf(END_TOKEN, p + BEGIN_TOKEN_SIZE); - if (s < 0) { - throw new RenderException("Missing '" + END_TOKEN + "': " + input); - } - - String expr = input.substring(p + BEGIN_TOKEN_SIZE, s); - - buff.append(input.substring(c, p)); - - evaluate(expr); - - c = s + END_TOKEN_SIZE; - } - - buff.append(input.substring(c)); - - return buff.toString(); - } - - private void evaluate(final String input) throws RenderException { - assert input != null; - - int i = input.indexOf(CODE_TEXT_SEPARATOR); - if (i < 0) { - throw new RenderException("Missing ANSI code/text separator '" + CODE_TEXT_SEPARATOR + "': " + input); - } - - String tmp = input.substring(0, i); - String[] codes = tmp.split(CODE_LIST_SEPARATOR); - String text = input.substring(i + 1, input.length()); - - for (int j = 0; j < codes.length; j++) { - int code = Code.forName(codes[j]); - buff.attrib(code); - } - - buff.append(text); - - buff.attrib(Code.OFF); - } - - // - // RenderException - // - - public static class RenderException - extends RuntimeException { - public RenderException(final String msg) { - super(msg); - } - } - - // - // Helpers - // - - public static boolean test(final String text) { - return text != null && text.indexOf(BEGIN_TOKEN) >= 0; - } - - public static String encode(final String text, final int code) { - return new StringBuffer(BEGIN_TOKEN). - append(Code.name(code)). - append(CODE_TEXT_SEPARATOR). - append(text). - append(END_TOKEN). - toString(); - } - } - - // - // RenderWriter - // - - public static class RenderWriter extends PrintWriter { - private final Renderer renderer = new Renderer(); - - public RenderWriter(final OutputStream out) { - super(out); - } - - public RenderWriter(final OutputStream out, final boolean autoFlush) { - super(out, autoFlush); - } - - public RenderWriter(final Writer out) { - super(out); - } - - public RenderWriter(final Writer out, final boolean autoFlush) { - super(out, autoFlush); - } - - public void write(final String s) { - if (Renderer.test(s)) { - super.write(renderer.render(s)); - } else { - super.write(s); - } - } - } -} diff --git a/src/main/java/org/elasticsearch/common/logging/log4j/JLinePatternLayout.java b/src/main/java/org/elasticsearch/common/logging/log4j/JLinePatternLayout.java deleted file mode 100644 index 606f36738d5..00000000000 --- a/src/main/java/org/elasticsearch/common/logging/log4j/JLinePatternLayout.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.logging.log4j; - -import org.apache.log4j.Level; -import org.apache.log4j.PatternLayout; -import org.apache.log4j.helpers.FormattingInfo; -import org.apache.log4j.helpers.PatternConverter; -import org.apache.log4j.helpers.PatternParser; -import org.apache.log4j.spi.LoggingEvent; -import org.elasticsearch.common.jline.ANSI; - -import java.lang.reflect.Field; - -import static jline.ANSIBuffer.ANSICodes.attrib; -import static org.elasticsearch.common.jline.ANSI.Code.*; - -/** - * - */ -public class JLinePatternLayout extends PatternLayout { - - @Override - protected PatternParser createPatternParser(String pattern) { - try { - return new JLinePatternParser(pattern); - } catch (Throwable t) { - return super.createPatternParser(pattern); - } - } - - private final static class JLinePatternParser extends PatternParser { - - private JLinePatternParser(String pattern) { - super(pattern); - } - - @Override - protected void addConverter(PatternConverter pc) { - try { - if (ANSI.isEnabled()) { - if (pc.getClass().getName().endsWith("BasicPatternConverter")) { - Field typeField = pc.getClass().getDeclaredField("type"); - typeField.setAccessible(true); - Integer type = (Integer) typeField.get(pc); - if (type == 2002) { - pc = new ColoredLevelPatternConverter(formattingInfo); - } - } - } - } catch (Throwable t) { - // ignore - } - super.addConverter(pc); - } - - private static class ColoredLevelPatternConverter extends PatternConverter { - - ColoredLevelPatternConverter(FormattingInfo formattingInfo) { - super(formattingInfo); - } - - public String convert(LoggingEvent event) { - if (!ANSI.isEnabled()) { - return event.getLevel().toString(); - } - if (event.getLevel() == Level.FATAL) { - return attrib(FG_RED) + event.getLevel().toString() + attrib(OFF); - } else if (event.getLevel() == Level.ERROR) { - return attrib(FG_RED) + event.getLevel().toString() + attrib(OFF); - } else if (event.getLevel() == Level.WARN) { - return attrib(FG_YELLOW) + event.getLevel().toString() + ' ' + attrib(OFF); - } else if (event.getLevel() == Level.INFO) { - return attrib(FG_GREEN) + event.getLevel().toString() + ' ' + attrib(OFF); - } else if (event.getLevel() == Level.DEBUG) { - return attrib(FG_CYAN) + event.getLevel().toString() + attrib(OFF); - } else if (event.getLevel() == Level.TRACE) { - return attrib(FG_BLUE) + event.getLevel().toString() + attrib(OFF); - } - return event.getLevel().toString(); - } - } - } -} diff --git a/src/main/java/org/elasticsearch/common/logging/log4j/LogConfigurator.java b/src/main/java/org/elasticsearch/common/logging/log4j/LogConfigurator.java index 9d6eccb1156..5f70328d6ff 100644 --- a/src/main/java/org/elasticsearch/common/logging/log4j/LogConfigurator.java +++ b/src/main/java/org/elasticsearch/common/logging/log4j/LogConfigurator.java @@ -60,7 +60,7 @@ public class LogConfigurator { .put("simple", "org.apache.log4j.SimpleLayout") .put("html", "org.apache.log4j.HTMLLayout") .put("pattern", "org.apache.log4j.PatternLayout") - .put("consolePattern", "org.elasticsearch.common.logging.log4j.JLinePatternLayout") + .put("consolePattern", "org.apache.log4j.PatternLayout") .put("ttcc", "org.apache.log4j.TTCCLayout") .put("xml", "org.apache.log4j.XMLLayout") .immutableMap(); From c53f9694d1aa2f1c6d31a49dadf91e1da53308c7 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 16 Dec 2011 15:43:15 +0200 Subject: [PATCH 040/270] add a utility main class that will automatically run it in the foreground without setting a system property --- .../elasticsearch/bootstrap/ElasticSearchF.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/main/java/org/elasticsearch/bootstrap/ElasticSearchF.java diff --git a/src/main/java/org/elasticsearch/bootstrap/ElasticSearchF.java b/src/main/java/org/elasticsearch/bootstrap/ElasticSearchF.java new file mode 100644 index 00000000000..8a9a7f68985 --- /dev/null +++ b/src/main/java/org/elasticsearch/bootstrap/ElasticSearchF.java @@ -0,0 +1,17 @@ +package org.elasticsearch.bootstrap; + +/** + * Same as {@link ElasticSearch} just runs it in the foreground by default (does not close + * sout and serr). + */ +public class ElasticSearchF { + + public static void close(String[] args) { + Bootstrap.close(args); + } + + public static void main(String[] args) { + System.setProperty("es-foreground", "yes"); + Bootstrap.main(args); + } +} From 3ddbc6469a3f2e57255dafaa4cf519ea3802f9b9 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 16 Dec 2011 17:01:02 +0200 Subject: [PATCH 041/270] fix spelling --- .../java/org/elasticsearch/index/mapper/core/TypeParsers.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java b/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java index d6735dd9963..0c77e836468 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java @@ -149,7 +149,7 @@ public class TypeParsers { } else if ("full".equals(path)) { return ContentPath.Type.FULL; } else { - throw new MapperParsingException("Wrong value for pathType [" + path + "] for objet [" + name + "]"); + throw new MapperParsingException("Wrong value for pathType [" + path + "] for object [" + name + "]"); } } From f74256afcd635d9db9b27f18808a09eb1e3f8727 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 16 Dec 2011 18:29:10 +0200 Subject: [PATCH 042/270] improve multicast message to have an internal header for node to node discovery --- src/main/java/org/elasticsearch/Version.java | 2 +- .../zen/ping/multicast/MulticastZenPing.java | 39 ++++++++++++++----- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/elasticsearch/Version.java b/src/main/java/org/elasticsearch/Version.java index 4df0bded3f6..26481821c69 100644 --- a/src/main/java/org/elasticsearch/Version.java +++ b/src/main/java/org/elasticsearch/Version.java @@ -58,7 +58,7 @@ public class Version { return fromId(in.readVInt()); } - private static Version fromId(int id) { + public static Version fromId(int id) { switch (id) { case V_0_18_0_ID: return V_0_18_0; diff --git a/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java b/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java index 356c60cc7f8..9c042216cf4 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java @@ -58,6 +58,8 @@ import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadF */ public class MulticastZenPing extends AbstractLifecycleComponent implements ZenPing { + private static final byte[] INTERNAL_HEADER = new byte[]{1, 9, 8, 4}; + private final String address; private final int port; @@ -267,6 +269,8 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry(); try { HandlesStreamOutput out = cachedEntry.cachedHandlesBytes(); + out.writeBytes(INTERNAL_HEADER); + Version.writeVersion(Version.CURRENT, out); out.writeInt(id); clusterName.writeTo(out); nodesProvider.nodes().localNode().writeTo(out); @@ -365,7 +369,7 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem ClusterName clusterName = null; Map externalPingData = null; - XContentType xContentType; + XContentType xContentType = null; synchronized (receiveMutex) { try { @@ -379,17 +383,34 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem continue; } try { - xContentType = XContentFactory.xContentType(datagramPacketReceive.getData(), datagramPacketReceive.getOffset(), datagramPacketReceive.getLength()); - if (xContentType != null) { - // an external ping - externalPingData = XContentFactory.xContent(xContentType) - .createParser(datagramPacketReceive.getData(), datagramPacketReceive.getOffset(), datagramPacketReceive.getLength()) - .mapAndClose(); - } else { - StreamInput input = CachedStreamInput.cachedHandles(new BytesStreamInput(datagramPacketReceive.getData(), datagramPacketReceive.getOffset(), datagramPacketReceive.getLength())); + boolean internal = false; + if (datagramPacketReceive.getLength() > 4) { + int counter = 0; + for (; counter < INTERNAL_HEADER.length; counter++) { + if (datagramPacketReceive.getData()[datagramPacketReceive.getOffset() + counter] != INTERNAL_HEADER[counter]) { + break; + } + } + if (counter == INTERNAL_HEADER.length) { + internal = true; + } + } + if (internal) { + StreamInput input = CachedStreamInput.cachedHandles(new BytesStreamInput(datagramPacketReceive.getData(), datagramPacketReceive.getOffset() + INTERNAL_HEADER.length, datagramPacketReceive.getLength())); + Version version = Version.readVersion(input); id = input.readInt(); clusterName = ClusterName.readClusterName(input); requestingNodeX = readNode(input); + } else { + xContentType = XContentFactory.xContentType(datagramPacketReceive.getData(), datagramPacketReceive.getOffset(), datagramPacketReceive.getLength()); + if (xContentType != null) { + // an external ping + externalPingData = XContentFactory.xContent(xContentType) + .createParser(datagramPacketReceive.getData(), datagramPacketReceive.getOffset(), datagramPacketReceive.getLength()) + .mapAndClose(); + } else { + throw new ElasticSearchIllegalStateException("failed multicast message, probably message from previous version"); + } } } catch (Exception e) { logger.warn("failed to read requesting data from {}", e, datagramPacketReceive.getSocketAddress()); From cb4c249a659b0ecee292f45c7f00aefd8ca0411e Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 16 Dec 2011 19:17:31 +0200 Subject: [PATCH 043/270] Search: When searching against a type with a dfs search type, dfs is ignored, closes #1546. --- .../search/highlight/HighlightPhase.java | 26 +++++++++++++------ .../search/query/QueryPhase.java | 8 +++--- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java b/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java index 9c34a11949b..fbf4fd21323 100644 --- a/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java +++ b/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java @@ -127,16 +127,26 @@ public class HighlightPhase implements FetchSubPhase { // QueryScorer uses WeightedSpanTermExtractor to extract terms, but we can't really plug into // it, so, we hack here (and really only support top level queries) Query query = context.parsedQuery().query(); - if (query instanceof FunctionScoreQuery) { - query = ((FunctionScoreQuery) query).getSubQuery(); - } else if (query instanceof FiltersFunctionScoreQuery) { - query = ((FiltersFunctionScoreQuery) query).getSubQuery(); - } else if (query instanceof ConstantScoreQuery) { - ConstantScoreQuery q = (ConstantScoreQuery) query; - if (q.getQuery() != null) { - query = q.getQuery(); + while (true) { + boolean extracted = false; + if (query instanceof FunctionScoreQuery) { + query = ((FunctionScoreQuery) query).getSubQuery(); + extracted = true; + } else if (query instanceof FiltersFunctionScoreQuery) { + query = ((FiltersFunctionScoreQuery) query).getSubQuery(); + extracted = true; + } else if (query instanceof ConstantScoreQuery) { + ConstantScoreQuery q = (ConstantScoreQuery) query; + if (q.getQuery() != null) { + query = q.getQuery(); + extracted = true; + } + } + if (!extracted) { + break; } } + QueryScorer queryScorer = new QueryScorer(query, null); queryScorer.setExpandMultiTermQuery(true); Fragmenter fragmenter; diff --git a/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 04d6b8aaa80..21dffcb6001 100644 --- a/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -81,6 +81,10 @@ public class QueryPhase implements SearchPhase { if (context.queryBoost() != 1.0f) { context.parsedQuery(new ParsedQuery(new FunctionScoreQuery(context.query(), new BoostScoreFunction(context.queryBoost())), context.parsedQuery())); } + Filter searchFilter = context.mapperService().searchFilter(context.types()); + if (searchFilter != null) { + context.parsedQuery(new ParsedQuery(new FilteredQuery(context.query(), context.filterCache().cache(searchFilter)), context.parsedQuery())); + } facetPhase.preProcess(context); } @@ -165,10 +169,6 @@ public class QueryPhase implements SearchPhase { searchContext.queryResult().size(searchContext.size()); Query query = searchContext.query(); - Filter searchFilter = searchContext.mapperService().searchFilter(searchContext.types()); - if (searchFilter != null) { - query = new FilteredQuery(query, searchContext.filterCache().cache(searchFilter)); - } TopDocs topDocs; int numDocs = searchContext.from() + searchContext.size(); From 9c1280f6eb8599acda3932a70c330a61165bf494 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 16 Dec 2011 22:55:00 +0200 Subject: [PATCH 044/270] add future support to track current merge --- .../TrackingConcurrentMergeScheduler.java | 2 ++ .../lucene/index/TrackingMergeScheduler.java | 24 +++++++++++++++++++ .../index/TrackingSerialMergeScheduler.java | 2 ++ 3 files changed, 28 insertions(+) create mode 100644 src/main/java/org/apache/lucene/index/TrackingMergeScheduler.java diff --git a/src/main/java/org/apache/lucene/index/TrackingConcurrentMergeScheduler.java b/src/main/java/org/apache/lucene/index/TrackingConcurrentMergeScheduler.java index cd0799a9798..bd6c7d497be 100644 --- a/src/main/java/org/apache/lucene/index/TrackingConcurrentMergeScheduler.java +++ b/src/main/java/org/apache/lucene/index/TrackingConcurrentMergeScheduler.java @@ -86,8 +86,10 @@ public class TrackingConcurrentMergeScheduler extends ConcurrentMergeScheduler { logger.trace("merge [{}] starting...", merge.info.name); } try { + TrackingMergeScheduler.setCurrentMerge(merge); super.doMerge(merge); } finally { + TrackingMergeScheduler.removeCurrentMerge(); long took = System.currentTimeMillis() - time; currentMerges.dec(); diff --git a/src/main/java/org/apache/lucene/index/TrackingMergeScheduler.java b/src/main/java/org/apache/lucene/index/TrackingMergeScheduler.java new file mode 100644 index 00000000000..cd09e513df4 --- /dev/null +++ b/src/main/java/org/apache/lucene/index/TrackingMergeScheduler.java @@ -0,0 +1,24 @@ +package org.apache.lucene.index; + +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; + +import java.util.concurrent.ConcurrentMap; + +/** + */ +public class TrackingMergeScheduler { + + private static final ConcurrentMap merges = ConcurrentCollections.newConcurrentMap(); + + public static void setCurrentMerge(MergePolicy.OneMerge merge) { + merges.put(Thread.currentThread(), merge); + } + + public static void removeCurrentMerge() { + merges.remove(Thread.currentThread()); + } + + public static MergePolicy.OneMerge getCurrentMerge() { + return merges.get(Thread.currentThread()); + } +} diff --git a/src/main/java/org/apache/lucene/index/TrackingSerialMergeScheduler.java b/src/main/java/org/apache/lucene/index/TrackingSerialMergeScheduler.java index b1a34c954f3..6cef4553d62 100644 --- a/src/main/java/org/apache/lucene/index/TrackingSerialMergeScheduler.java +++ b/src/main/java/org/apache/lucene/index/TrackingSerialMergeScheduler.java @@ -93,8 +93,10 @@ public class TrackingSerialMergeScheduler extends MergeScheduler { currentMergesNumDocs.inc(totalNumDocs); currentMergesSizeInBytes.inc(totalSizeInBytes); try { + TrackingMergeScheduler.setCurrentMerge(merge); writer.merge(merge); } finally { + TrackingMergeScheduler.removeCurrentMerge(); long took = System.currentTimeMillis() - time; currentMerges.dec(); From 26fc9bcb251034d183be0b3c5b516163af8ec012 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 16 Dec 2011 23:30:31 +0200 Subject: [PATCH 045/270] abstract away the fs translog file to an interface --- .../index/translog/fs/FsTranslog.java | 4 +- .../index/translog/fs/FsTranslogFile.java | 82 ++----------- .../translog/fs/SimpleFsTranslogFile.java | 108 ++++++++++++++++++ .../stress/SingleThreadBulkStress.java | 6 +- 4 files changed, 121 insertions(+), 79 deletions(-) create mode 100644 src/main/java/org/elasticsearch/index/translog/fs/SimpleFsTranslogFile.java diff --git a/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java b/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java index 746c75bcca2..d31f21e9efb 100644 --- a/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java +++ b/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java @@ -149,7 +149,7 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog } } try { - newFile = new FsTranslogFile(shardId, id, new RafReference(new File(location, "translog-" + id))); + newFile = new SimpleFsTranslogFile(shardId, id, new RafReference(new File(location, "translog-" + id))); } catch (IOException e) { throw new TranslogException(shardId, "failed to create new translog file", e); } @@ -184,7 +184,7 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog location = file; } } - this.trans = new FsTranslogFile(shardId, id, new RafReference(new File(location, "translog-" + id))); + this.trans = new SimpleFsTranslogFile(shardId, id, new RafReference(new File(location, "translog-" + id))); } catch (IOException e) { throw new TranslogException(shardId, "failed to create new translog file", e); } finally { diff --git a/src/main/java/org/elasticsearch/index/translog/fs/FsTranslogFile.java b/src/main/java/org/elasticsearch/index/translog/fs/FsTranslogFile.java index 4c4c5c4dccd..b774eba83ae 100644 --- a/src/main/java/org/elasticsearch/index/translog/fs/FsTranslogFile.java +++ b/src/main/java/org/elasticsearch/index/translog/fs/FsTranslogFile.java @@ -19,90 +19,26 @@ package org.elasticsearch.index.translog.fs; -import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogException; import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -public class FsTranslogFile { +public interface FsTranslogFile { - private final long id; - private final ShardId shardId; - private final RafReference raf; + long id(); - private final AtomicInteger operationCounter = new AtomicInteger(); + int estimatedNumberOfOperations(); - private final AtomicLong lastPosition = new AtomicLong(0); - private final AtomicLong lastWrittenPosition = new AtomicLong(0); + long translogSizeInBytes(); - private volatile long lastSyncPosition = 0; + Translog.Location add(byte[] data, int from, int size) throws IOException; - public FsTranslogFile(ShardId shardId, long id, RafReference raf) throws IOException { - this.shardId = shardId; - this.id = id; - this.raf = raf; - raf.raf().setLength(0); - } + byte[] read(Translog.Location location) throws IOException; - public long id() { - return this.id; - } + void close(boolean delete); - public int estimatedNumberOfOperations() { - return operationCounter.get(); - } + FsChannelSnapshot snapshot() throws TranslogException; - public long translogSizeInBytes() { - return lastWrittenPosition.get(); - } - - public Translog.Location add(byte[] data, int from, int size) throws IOException { - long position = lastPosition.getAndAdd(size); - raf.channel().write(ByteBuffer.wrap(data, from, size), position); - lastWrittenPosition.getAndAdd(size); - operationCounter.incrementAndGet(); - return new Translog.Location(id, position, size); - } - - public byte[] read(Translog.Location location) throws IOException { - ByteBuffer buffer = ByteBuffer.allocate(location.size); - raf.channel().read(buffer, location.translogLocation); - return buffer.array(); - } - - public void close(boolean delete) { - raf.decreaseRefCount(delete); - } - - /** - * Returns a snapshot on this file, null if it failed to snapshot. - */ - public FsChannelSnapshot snapshot() throws TranslogException { - try { - if (!raf.increaseRefCount()) { - return null; - } - return new FsChannelSnapshot(this.id, raf, lastWrittenPosition.get(), operationCounter.get()); - } catch (Exception e) { - throw new TranslogException(shardId, "Failed to snapshot", e); - } - } - - public void sync() { - try { - // check if we really need to sync here... - long last = lastWrittenPosition.get(); - if (last == lastSyncPosition) { - return; - } - lastSyncPosition = last; - raf.channel().force(false); - } catch (Exception e) { - // ignore - } - } + void sync(); } diff --git a/src/main/java/org/elasticsearch/index/translog/fs/SimpleFsTranslogFile.java b/src/main/java/org/elasticsearch/index/translog/fs/SimpleFsTranslogFile.java new file mode 100644 index 00000000000..f9a3619f987 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/translog/fs/SimpleFsTranslogFile.java @@ -0,0 +1,108 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.translog.fs; + +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.index.translog.TranslogException; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class SimpleFsTranslogFile implements FsTranslogFile { + + private final long id; + private final ShardId shardId; + private final RafReference raf; + + private final AtomicInteger operationCounter = new AtomicInteger(); + + private final AtomicLong lastPosition = new AtomicLong(0); + private final AtomicLong lastWrittenPosition = new AtomicLong(0); + + private volatile long lastSyncPosition = 0; + + public SimpleFsTranslogFile(ShardId shardId, long id, RafReference raf) throws IOException { + this.shardId = shardId; + this.id = id; + this.raf = raf; + raf.raf().setLength(0); + } + + public long id() { + return this.id; + } + + public int estimatedNumberOfOperations() { + return operationCounter.get(); + } + + public long translogSizeInBytes() { + return lastWrittenPosition.get(); + } + + public Translog.Location add(byte[] data, int from, int size) throws IOException { + long position = lastPosition.getAndAdd(size); + raf.channel().write(ByteBuffer.wrap(data, from, size), position); + lastWrittenPosition.getAndAdd(size); + operationCounter.incrementAndGet(); + return new Translog.Location(id, position, size); + } + + public byte[] read(Translog.Location location) throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(location.size); + raf.channel().read(buffer, location.translogLocation); + return buffer.array(); + } + + public void close(boolean delete) { + raf.decreaseRefCount(delete); + } + + /** + * Returns a snapshot on this file, null if it failed to snapshot. + */ + public FsChannelSnapshot snapshot() throws TranslogException { + try { + if (!raf.increaseRefCount()) { + return null; + } + return new FsChannelSnapshot(this.id, raf, lastWrittenPosition.get(), operationCounter.get()); + } catch (Exception e) { + throw new TranslogException(shardId, "Failed to snapshot", e); + } + } + + public void sync() { + try { + // check if we really need to sync here... + long last = lastWrittenPosition.get(); + if (last == lastSyncPosition) { + return; + } + lastSyncPosition = last; + raf.channel().force(false); + } catch (Exception e) { + // ignore + } + } +} diff --git a/src/test/java/org/elasticsearch/benchmark/stress/SingleThreadBulkStress.java b/src/test/java/org/elasticsearch/benchmark/stress/SingleThreadBulkStress.java index 5303955fa89..b1c8ee1c545 100644 --- a/src/test/java/org/elasticsearch/benchmark/stress/SingleThreadBulkStress.java +++ b/src/test/java/org/elasticsearch/benchmark/stress/SingleThreadBulkStress.java @@ -32,7 +32,6 @@ import org.elasticsearch.node.Node; import java.io.IOException; import java.util.Random; -import java.util.concurrent.TimeUnit; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; @@ -50,11 +49,10 @@ public class SingleThreadBulkStress { Random random = new Random(); Settings settings = settingsBuilder() - .put("cluster.routing.schedule", 200, TimeUnit.MILLISECONDS) - .put("index.refresh_interval", "-1") + .put("index.refresh_interval", "1s") .put("index.merge.async", true) .put("index.translog.flush_threshold_ops", 5000) - .put("gateway.type", "local") + .put("gateway.type", "none") .put(SETTING_NUMBER_OF_SHARDS, 1) .put(SETTING_NUMBER_OF_REPLICAS, 1) .build(); From ec04435b06dc3eab8033e27c66028348317e1e72 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 16 Dec 2011 23:36:31 +0200 Subject: [PATCH 046/270] rename test --- ...impleTranslogTests.java => FsSimpleTranslogTests.java} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename src/test/java/org/elasticsearch/test/unit/index/translog/fs/{FsChannelSimpleTranslogTests.java => FsSimpleTranslogTests.java} (89%) diff --git a/src/test/java/org/elasticsearch/test/unit/index/translog/fs/FsChannelSimpleTranslogTests.java b/src/test/java/org/elasticsearch/test/unit/index/translog/fs/FsSimpleTranslogTests.java similarity index 89% rename from src/test/java/org/elasticsearch/test/unit/index/translog/fs/FsChannelSimpleTranslogTests.java rename to src/test/java/org/elasticsearch/test/unit/index/translog/fs/FsSimpleTranslogTests.java index cada35c8d19..cc4e3776cdc 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/translog/fs/FsChannelSimpleTranslogTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/translog/fs/FsSimpleTranslogTests.java @@ -20,9 +20,9 @@ package org.elasticsearch.test.unit.index.translog.fs; import org.elasticsearch.common.io.FileSystemUtils; -import org.elasticsearch.test.unit.index.translog.AbstractSimpleTranslogTests; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.fs.FsTranslog; +import org.elasticsearch.test.unit.index.translog.AbstractSimpleTranslogTests; import org.testng.annotations.AfterClass; import java.io.File; @@ -32,15 +32,15 @@ import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_ /** * */ -public class FsChannelSimpleTranslogTests extends AbstractSimpleTranslogTests { +public class FsSimpleTranslogTests extends AbstractSimpleTranslogTests { @Override protected Translog create() { - return new FsTranslog(shardId, EMPTY_SETTINGS, new File("work/fs-translog")); + return new FsTranslog(shardId, EMPTY_SETTINGS, new File("data/fs-translog")); } @AfterClass public void cleanup() { - FileSystemUtils.deleteRecursively(new File("work/fs-translog"), true); + FileSystemUtils.deleteRecursively(new File("data/fs-translog"), true); } } From 010b910333ee16f18b2a2ec81579bc2c3d9a0614 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 16 Dec 2011 23:50:11 +0200 Subject: [PATCH 047/270] 0.18.5 ignores query string analyzer, closes #1547. --- .../lucene/queryParser/MapperQueryParser.java | 19 +++++++++---- .../queryParser/QueryParserSettings.java | 27 ++++++++++++++----- .../index/query/FieldQueryParser.java | 6 ++--- .../index/query/QueryStringQueryParser.java | 6 ++--- 4 files changed, 38 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java b/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java index ff4c322aafa..a52194af67f 100644 --- a/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java +++ b/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java @@ -64,6 +64,8 @@ public class MapperQueryParser extends QueryParser { private final QueryParseContext parseContext; + private boolean forcedAnalyzer; + private FieldMapper currentMapper; private boolean analyzeWildcard; @@ -74,14 +76,15 @@ public class MapperQueryParser extends QueryParser { } public MapperQueryParser(QueryParserSettings settings, QueryParseContext parseContext) { - super(Lucene.QUERYPARSER_VERSION, settings.defaultField(), settings.analyzer()); + super(Lucene.QUERYPARSER_VERSION, settings.defaultField(), settings.defaultAnalyzer()); this.parseContext = parseContext; reset(settings); } public void reset(QueryParserSettings settings) { this.field = settings.defaultField(); - this.analyzer = settings.analyzer(); + this.forcedAnalyzer = settings.forcedAnalyzer() != null; + this.analyzer = forcedAnalyzer ? settings.forcedAnalyzer() : settings.defaultAnalyzer(); setMultiTermRewriteMethod(settings.rewriteMethod()); setEnablePositionIncrements(settings.enablePositionIncrements()); setAutoGeneratePhraseQueries(settings.autoGeneratePhraseQueries()); @@ -121,7 +124,9 @@ public class MapperQueryParser extends QueryParser { try { MapperService.SmartNameFieldMappers fieldMappers = parseContext.smartFieldMappers(field); if (fieldMappers != null) { - analyzer = fieldMappers.searchAnalyzer(); + if (!forcedAnalyzer) { + analyzer = fieldMappers.searchAnalyzer(); + } currentMapper = fieldMappers.fieldMappers().mapper(); if (currentMapper != null) { Query query = null; @@ -182,7 +187,9 @@ public class MapperQueryParser extends QueryParser { try { MapperService.SmartNameFieldMappers fieldMappers = parseContext.smartFieldMappers(field); if (fieldMappers != null) { - analyzer = fieldMappers.searchAnalyzer(); + if (!forcedAnalyzer) { + analyzer = fieldMappers.searchAnalyzer(); + } currentMapper = fieldMappers.fieldMappers().mapper(); if (currentMapper != null) { indexedNameField = currentMapper.names().indexName(); @@ -256,7 +263,9 @@ public class MapperQueryParser extends QueryParser { try { MapperService.SmartNameFieldMappers fieldMappers = parseContext.smartFieldMappers(field); if (fieldMappers != null) { - analyzer = fieldMappers.searchAnalyzer(); + if (!forcedAnalyzer) { + analyzer = fieldMappers.searchAnalyzer(); + } currentMapper = fieldMappers.fieldMappers().mapper(); if (currentMapper != null) { indexedNameField = currentMapper.names().indexName(); diff --git a/src/main/java/org/apache/lucene/queryParser/QueryParserSettings.java b/src/main/java/org/apache/lucene/queryParser/QueryParserSettings.java index eaef33cd479..110e02c9712 100644 --- a/src/main/java/org/apache/lucene/queryParser/QueryParserSettings.java +++ b/src/main/java/org/apache/lucene/queryParser/QueryParserSettings.java @@ -44,7 +44,8 @@ public class QueryParserSettings { private int fuzzyPrefixLength = FuzzyQuery.defaultPrefixLength; private boolean analyzeWildcard = DEFAULT_ANALYZE_WILDCARD; private boolean escape = false; - private Analyzer analyzer = null; + private Analyzer defaultAnalyzer = null; + private Analyzer forcedAnalyzer = null; private MultiTermQuery.RewriteMethod rewriteMethod = MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT; private String minimumShouldMatch; @@ -144,12 +145,20 @@ public class QueryParserSettings { this.escape = escape; } - public Analyzer analyzer() { - return analyzer; + public Analyzer defaultAnalyzer() { + return defaultAnalyzer; } - public void analyzer(Analyzer analyzer) { - this.analyzer = analyzer; + public void defaultAnalyzer(Analyzer defaultAnalyzer) { + this.defaultAnalyzer = defaultAnalyzer; + } + + public Analyzer forcedAnalyzer() { + return forcedAnalyzer; + } + + public void forcedAnalyzer(Analyzer forcedAnalyzer) { + this.forcedAnalyzer = forcedAnalyzer; } public boolean analyzeWildcard() { @@ -193,7 +202,10 @@ public class QueryParserSettings { if (fuzzyPrefixLength != that.fuzzyPrefixLength) return false; if (lowercaseExpandedTerms != that.lowercaseExpandedTerms) return false; if (phraseSlop != that.phraseSlop) return false; - if (analyzer != null ? !analyzer.equals(that.analyzer) : that.analyzer != null) return false; + if (defaultAnalyzer != null ? !defaultAnalyzer.equals(that.defaultAnalyzer) : that.defaultAnalyzer != null) + return false; + if (forcedAnalyzer != null ? !forcedAnalyzer.equals(that.forcedAnalyzer) : that.forcedAnalyzer != null) + return false; if (defaultField != null ? !defaultField.equals(that.defaultField) : that.defaultField != null) return false; if (defaultOperator != that.defaultOperator) return false; if (queryString != null ? !queryString.equals(that.queryString) : that.queryString != null) return false; @@ -219,7 +231,8 @@ public class QueryParserSettings { result = 31 * result + (fuzzyMinSim != +0.0f ? Float.floatToIntBits(fuzzyMinSim) : 0); result = 31 * result + fuzzyPrefixLength; result = 31 * result + (escape ? 1 : 0); - result = 31 * result + (analyzer != null ? analyzer.hashCode() : 0); + result = 31 * result + (defaultAnalyzer != null ? defaultAnalyzer.hashCode() : 0); + result = 31 * result + (forcedAnalyzer != null ? forcedAnalyzer.hashCode() : 0); result = 31 * result + (analyzeWildcard ? 1 : 0); return result; } diff --git a/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java b/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java index 2ba602ddcfa..a9e2e1606e3 100644 --- a/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java @@ -91,7 +91,7 @@ public class FieldQueryParser implements QueryParser { } else if ("phrase_slop".equals(currentFieldName) || "phraseSlop".equals(currentFieldName)) { qpSettings.phraseSlop(parser.intValue()); } else if ("analyzer".equals(currentFieldName)) { - qpSettings.analyzer(parseContext.analysisService().analyzer(parser.text())); + qpSettings.forcedAnalyzer(parseContext.analysisService().analyzer(parser.text())); } else if ("default_operator".equals(currentFieldName) || "defaultOperator".equals(currentFieldName)) { String op = parser.text(); if ("or".equalsIgnoreCase(op)) { @@ -123,9 +123,7 @@ public class FieldQueryParser implements QueryParser { parser.nextToken(); } - if (qpSettings.analyzer() == null) { - qpSettings.analyzer(parseContext.mapperService().searchAnalyzer()); - } + qpSettings.defaultAnalyzer(parseContext.mapperService().searchAnalyzer()); if (qpSettings.queryString() == null) { throw new QueryParsingException(parseContext.index(), "No value specified for term query"); diff --git a/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java b/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java index 7e2d7bd94bf..bb2d48f1e36 100644 --- a/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java @@ -135,7 +135,7 @@ public class QueryStringQueryParser implements QueryParser { throw new QueryParsingException(parseContext.index(), "Query default operator [" + op + "] is not allowed"); } } else if ("analyzer".equals(currentFieldName)) { - qpSettings.analyzer(parseContext.analysisService().analyzer(parser.text())); + qpSettings.forcedAnalyzer(parseContext.analysisService().analyzer(parser.text())); } else if ("allow_leading_wildcard".equals(currentFieldName) || "allowLeadingWildcard".equals(currentFieldName)) { qpSettings.allowLeadingWildcard(parser.booleanValue()); } else if ("auto_generate_phrase_queries".equals(currentFieldName) || "autoGeneratePhraseQueries".equals(currentFieldName)) { @@ -170,9 +170,7 @@ public class QueryStringQueryParser implements QueryParser { if (qpSettings.queryString() == null) { throw new QueryParsingException(parseContext.index(), "query_string must be provided with a [query]"); } - if (qpSettings.analyzer() == null) { - qpSettings.analyzer(parseContext.mapperService().searchAnalyzer()); - } + qpSettings.defaultAnalyzer(parseContext.mapperService().searchAnalyzer()); if (qpSettings.escape()) { qpSettings.queryString(org.apache.lucene.queryParser.QueryParser.escape(qpSettings.queryString())); From a3ca1afed5308277383eedf97bd209d5225c897a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 18 Dec 2011 00:19:35 +0200 Subject: [PATCH 048/270] Translog: When not sync'ing on each operation, buffer writes, closes #1549. --- .../org/elasticsearch/index/store/Store.java | 2 +- .../translog/fs/BufferingFsTranslogFile.java | 169 ++++++++++++++++++ .../index/translog/fs/FsTranslog.java | 100 +++++++++-- .../index/translog/fs/FsTranslogFile.java | 33 +++- .../translog/fs/SimpleFsTranslogFile.java | 5 + .../elasticsearch/indices/IndicesModule.java | 4 +- ...ler.java => IndexingMemoryController.java} | 43 +++-- .../node/internal/InternalNode.java | 8 +- .../stress/SingleThreadIndexingStress.java | 6 +- .../translog/AbstractSimpleTranslogTests.java | 19 ++ .../translog/fs/FsBufferedTranslogTests.java | 48 +++++ .../translog/fs/FsSimpleTranslogTests.java | 8 +- 12 files changed, 394 insertions(+), 51 deletions(-) create mode 100644 src/main/java/org/elasticsearch/index/translog/fs/BufferingFsTranslogFile.java rename src/main/java/org/elasticsearch/indices/memory/{IndexingMemoryBufferController.java => IndexingMemoryController.java} (88%) create mode 100644 src/test/java/org/elasticsearch/test/unit/index/translog/fs/FsBufferedTranslogTests.java diff --git a/src/main/java/org/elasticsearch/index/store/Store.java b/src/main/java/org/elasticsearch/index/store/Store.java index 3af35610582..1e5b402518d 100644 --- a/src/main/java/org/elasticsearch/index/store/Store.java +++ b/src/main/java/org/elasticsearch/index/store/Store.java @@ -274,7 +274,7 @@ public class Store extends AbstractIndexShardComponent { /** * The idea of the store directory is to cache file level meta data, as well as md5 of it */ - protected class StoreDirectory extends Directory implements ForceSyncDirectory { + class StoreDirectory extends Directory implements ForceSyncDirectory { private final Directory[] delegates; diff --git a/src/main/java/org/elasticsearch/index/translog/fs/BufferingFsTranslogFile.java b/src/main/java/org/elasticsearch/index/translog/fs/BufferingFsTranslogFile.java new file mode 100644 index 00000000000..7809c78bf12 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/translog/fs/BufferingFsTranslogFile.java @@ -0,0 +1,169 @@ +package org.elasticsearch.index.translog.fs; + +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.index.translog.TranslogException; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + */ +public class BufferingFsTranslogFile implements FsTranslogFile { + + private final long id; + private final ShardId shardId; + private final RafReference raf; + + private final ReadWriteLock rwl = new ReentrantReadWriteLock(); + + private volatile int operationCounter; + + private long lastPosition; + private volatile long lastWrittenPosition; + + private volatile long lastSyncPosition = 0; + + private byte[] buffer; + private int bufferCount; + + public BufferingFsTranslogFile(ShardId shardId, long id, RafReference raf, int bufferSize) throws IOException { + this.shardId = shardId; + this.id = id; + this.raf = raf; + this.buffer = new byte[bufferSize]; + raf.raf().setLength(0); + } + + public long id() { + return this.id; + } + + public int estimatedNumberOfOperations() { + return operationCounter; + } + + public long translogSizeInBytes() { + return lastWrittenPosition; + } + + @Override + public Translog.Location add(byte[] data, int from, int size) throws IOException { + rwl.writeLock().lock(); + try { + operationCounter++; + long position = lastPosition; + if (size >= buffer.length) { + flushBuffer(); + raf.raf().write(data, from, size); + lastWrittenPosition += size; + lastPosition += size; + return new Translog.Location(id, position, size); + } + if (size > buffer.length - bufferCount) { + flushBuffer(); + } + System.arraycopy(data, from, buffer, bufferCount, size); + bufferCount += size; + lastPosition += size; + return new Translog.Location(id, position, size); + } finally { + rwl.writeLock().unlock(); + } + } + + private void flushBuffer() throws IOException { + if (bufferCount > 0) { + raf.raf().write(buffer, 0, bufferCount); + lastWrittenPosition += bufferCount; + bufferCount = 0; + } + } + + @Override + public byte[] read(Translog.Location location) throws IOException { + rwl.readLock().lock(); + try { + if (location.translogLocation >= lastWrittenPosition) { + byte[] data = new byte[location.size]; + System.arraycopy(buffer, (int) (location.translogLocation - lastWrittenPosition), data, 0, location.size); + return data; + } + } finally { + rwl.readLock().unlock(); + } + ByteBuffer buffer = ByteBuffer.allocate(location.size); + raf.channel().read(buffer, location.translogLocation); + return buffer.array(); + } + + @Override + public FsChannelSnapshot snapshot() throws TranslogException { + rwl.writeLock().lock(); + try { + flushBuffer(); + if (!raf.increaseRefCount()) { + return null; + } + return new FsChannelSnapshot(this.id, raf, lastWrittenPosition, operationCounter); + } catch (IOException e) { + throw new TranslogException(shardId, "failed to flush", e); + } finally { + rwl.writeLock().unlock(); + } + } + + @Override + public void sync() { + try { + // check if we really need to sync here... + long last = lastWrittenPosition; + if (last == lastSyncPosition) { + return; + } + lastSyncPosition = last; + rwl.writeLock().lock(); + try { + flushBuffer(); + } finally { + rwl.writeLock().unlock(); + } + raf.channel().force(false); + } catch (Exception e) { + // ignore + } + } + + @Override + public void close(boolean delete) { + if (!delete) { + rwl.writeLock().lock(); + try { + flushBuffer(); + } catch (IOException e) { + throw new TranslogException(shardId, "failed to close", e); + } finally { + rwl.writeLock().unlock(); + } + } + raf.decreaseRefCount(delete); + } + + @Override + public void reuse(FsTranslogFile other) { + if (!(other instanceof BufferingFsTranslogFile)) { + return; + } + rwl.writeLock().lock(); + try { + flushBuffer(); + this.buffer = ((BufferingFsTranslogFile) other).buffer; + } catch (IOException e) { + throw new TranslogException(shardId, "failed to flush", e); + } finally { + rwl.writeLock().unlock(); + } + } +} diff --git a/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java b/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java index d31f21e9efb..23a4f6d6abd 100644 --- a/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java +++ b/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java @@ -20,13 +20,16 @@ package org.elasticsearch.index.translog.fs; import jsr166y.ThreadLocalRandom; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.CachedStreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.settings.IndexSettingsService; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.translog.Translog; @@ -44,29 +47,99 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; */ public class FsTranslog extends AbstractIndexShardComponent implements Translog { + static { + IndexMetaData.addDynamicSettings( + "index.translog.fs.type", + "index.translog.fs.buffer_size", + "index.translog.fs.transient_buffer_size" + ); + } + + class ApplySettings implements IndexSettingsService.Listener { + @Override + public void onRefreshSettings(Settings settings) { + int bufferSize = (int) settings.getAsBytesSize("index.translog.fs.buffer_size", new ByteSizeValue(FsTranslog.this.bufferSize)).bytes(); + if (bufferSize != FsTranslog.this.bufferSize) { + logger.info("updating buffer_size from [{}] to [{}]", new ByteSizeValue(FsTranslog.this.bufferSize), new ByteSizeValue(bufferSize)); + FsTranslog.this.bufferSize = bufferSize; + } + + int transientBufferSize = (int) settings.getAsBytesSize("index.translog.fs.transient_buffer_size", new ByteSizeValue(FsTranslog.this.transientBufferSize)).bytes(); + if (transientBufferSize != FsTranslog.this.transientBufferSize) { + logger.info("updating transient_buffer_size from [{}] to [{}]", new ByteSizeValue(FsTranslog.this.transientBufferSize), new ByteSizeValue(transientBufferSize)); + FsTranslog.this.transientBufferSize = transientBufferSize; + } + + FsTranslogFile.Type type = FsTranslogFile.Type.fromString(settings.get("index.translog.fs.type", FsTranslog.this.type.name())); + if (type != FsTranslog.this.type) { + logger.info("updating type from [{}] to [{}]", FsTranslog.this.type, type); + FsTranslog.this.type = type; + } + } + } + + private final IndexSettingsService indexSettingsService; + private final ReadWriteLock rwl = new ReentrantReadWriteLock(); private final File[] locations; private volatile FsTranslogFile current; private volatile FsTranslogFile trans; + private FsTranslogFile.Type type; + private boolean syncOnEachOperation = false; + private int bufferSize; + private int transientBufferSize; + + private final ApplySettings applySettings = new ApplySettings(); + @Inject - public FsTranslog(ShardId shardId, @IndexSettings Settings indexSettings, NodeEnvironment nodeEnv) { + public FsTranslog(ShardId shardId, @IndexSettings Settings indexSettings, IndexSettingsService indexSettingsService, NodeEnvironment nodeEnv) { super(shardId, indexSettings); + this.indexSettingsService = indexSettingsService; File[] shardLocations = nodeEnv.shardLocations(shardId); this.locations = new File[shardLocations.length]; for (int i = 0; i < shardLocations.length; i++) { locations[i] = new File(shardLocations[i], "translog"); FileSystemUtils.mkdirs(locations[i]); } + + this.type = FsTranslogFile.Type.fromString(componentSettings.get("type", FsTranslogFile.Type.BUFFERED.name())); + this.bufferSize = (int) componentSettings.getAsBytesSize("buffer_size", ByteSizeValue.parseBytesSizeValue("64k")).bytes(); + this.transientBufferSize = (int) componentSettings.getAsBytesSize("transient_buffer_size", ByteSizeValue.parseBytesSizeValue("8k")).bytes(); + + indexSettingsService.addListener(applySettings); } public FsTranslog(ShardId shardId, @IndexSettings Settings indexSettings, File location) { super(shardId, indexSettings); + this.indexSettingsService = null; this.locations = new File[]{location}; FileSystemUtils.mkdirs(location); + + this.type = FsTranslogFile.Type.fromString(componentSettings.get("type", FsTranslogFile.Type.BUFFERED.name())); + } + + @Override + public void close(boolean delete) { + if (indexSettingsService != null) { + indexSettingsService.removeListener(applySettings); + } + rwl.writeLock().lock(); + try { + FsTranslogFile current1 = this.current; + if (current1 != null) { + current1.close(delete); + } + current1 = this.trans; + if (current1 != null) { + current1.close(delete); + } + } finally { + rwl.writeLock().unlock(); + } } public File[] locations() { @@ -149,7 +222,7 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog } } try { - newFile = new SimpleFsTranslogFile(shardId, id, new RafReference(new File(location, "translog-" + id))); + newFile = type.create(shardId, id, new RafReference(new File(location, "translog-" + id)), bufferSize); } catch (IOException e) { throw new TranslogException(shardId, "failed to create new translog file", e); } @@ -184,7 +257,7 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog location = file; } } - this.trans = new SimpleFsTranslogFile(shardId, id, new RafReference(new File(location, "translog-" + id))); + this.trans = type.create(shardId, id, new RafReference(new File(location, "translog-" + id)), transientBufferSize); } catch (IOException e) { throw new TranslogException(shardId, "failed to create new translog file", e); } finally { @@ -205,6 +278,7 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog rwl.writeLock().unlock(); } old.close(true); + current.reuse(old); } @Override @@ -311,22 +385,10 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog @Override public void syncOnEachOperation(boolean syncOnEachOperation) { this.syncOnEachOperation = syncOnEachOperation; - } - - @Override - public void close(boolean delete) { - rwl.writeLock().lock(); - try { - FsTranslogFile current1 = this.current; - if (current1 != null) { - current1.close(delete); - } - current1 = this.trans; - if (current1 != null) { - current1.close(delete); - } - } finally { - rwl.writeLock().unlock(); + if (syncOnEachOperation) { + type = FsTranslogFile.Type.SIMPLE; + } else { + type = FsTranslogFile.Type.BUFFERED; } } } diff --git a/src/main/java/org/elasticsearch/index/translog/fs/FsTranslogFile.java b/src/main/java/org/elasticsearch/index/translog/fs/FsTranslogFile.java index b774eba83ae..68a70130310 100644 --- a/src/main/java/org/elasticsearch/index/translog/fs/FsTranslogFile.java +++ b/src/main/java/org/elasticsearch/index/translog/fs/FsTranslogFile.java @@ -19,6 +19,8 @@ package org.elasticsearch.index.translog.fs; +import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogException; @@ -26,6 +28,33 @@ import java.io.IOException; public interface FsTranslogFile { + public static enum Type { + + SIMPLE() { + @Override + public FsTranslogFile create(ShardId shardId, long id, RafReference raf, int bufferSize) throws IOException { + return new SimpleFsTranslogFile(shardId, id, raf); + } + }, + BUFFERED() { + @Override + public FsTranslogFile create(ShardId shardId, long id, RafReference raf, int bufferSize) throws IOException { + return new BufferingFsTranslogFile(shardId, id, raf, bufferSize); + } + }; + + public abstract FsTranslogFile create(ShardId shardId, long id, RafReference raf, int bufferSize) throws IOException; + + public static Type fromString(String type) throws ElasticSearchIllegalArgumentException { + if (SIMPLE.name().equalsIgnoreCase(type)) { + return SIMPLE; + } else if (BUFFERED.name().equalsIgnoreCase(type)) { + return BUFFERED; + } + throw new ElasticSearchIllegalArgumentException("No translog fs type [" + type + "]"); + } + } + long id(); int estimatedNumberOfOperations(); @@ -36,9 +65,11 @@ public interface FsTranslogFile { byte[] read(Translog.Location location) throws IOException; - void close(boolean delete); + void close(boolean delete) throws TranslogException; FsChannelSnapshot snapshot() throws TranslogException; + void reuse(FsTranslogFile other) throws TranslogException; + void sync(); } diff --git a/src/main/java/org/elasticsearch/index/translog/fs/SimpleFsTranslogFile.java b/src/main/java/org/elasticsearch/index/translog/fs/SimpleFsTranslogFile.java index f9a3619f987..3a64f219d88 100644 --- a/src/main/java/org/elasticsearch/index/translog/fs/SimpleFsTranslogFile.java +++ b/src/main/java/org/elasticsearch/index/translog/fs/SimpleFsTranslogFile.java @@ -105,4 +105,9 @@ public class SimpleFsTranslogFile implements FsTranslogFile { // ignore } } + + @Override + public void reuse(FsTranslogFile other) { + // nothing to do there + } } diff --git a/src/main/java/org/elasticsearch/indices/IndicesModule.java b/src/main/java/org/elasticsearch/indices/IndicesModule.java index dd8ba0086fa..1aa5c91264f 100644 --- a/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -27,7 +27,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.analysis.IndicesAnalysisModule; import org.elasticsearch.indices.cache.filter.IndicesNodeFilterCache; import org.elasticsearch.indices.cluster.IndicesClusterStateService; -import org.elasticsearch.indices.memory.IndexingMemoryBufferController; +import org.elasticsearch.indices.memory.IndexingMemoryController; import org.elasticsearch.indices.query.IndicesQueriesModule; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.indices.recovery.RecoverySource; @@ -62,7 +62,7 @@ public class IndicesModule extends AbstractModule implements SpawnModules { bind(RecoverySource.class).asEagerSingleton(); bind(IndicesClusterStateService.class).asEagerSingleton(); - bind(IndexingMemoryBufferController.class).asEagerSingleton(); + bind(IndexingMemoryController.class).asEagerSingleton(); bind(IndicesNodeFilterCache.class).asEagerSingleton(); bind(TransportNodesListShardStoreMetaData.class).asEagerSingleton(); bind(IndicesTTLService.class).asEagerSingleton(); diff --git a/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryBufferController.java b/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java similarity index 88% rename from src/main/java/org/elasticsearch/indices/memory/IndexingMemoryBufferController.java rename to src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java index 71aec5faa82..5b3b9889a3a 100644 --- a/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryBufferController.java +++ b/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java @@ -19,6 +19,7 @@ package org.elasticsearch.indices.memory; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.common.component.AbstractLifecycleComponent; @@ -40,13 +41,14 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.threadpool.ThreadPool; +import java.util.List; import java.util.Map; import java.util.concurrent.ScheduledFuture; /** * */ -public class IndexingMemoryBufferController extends AbstractLifecycleComponent { +public class IndexingMemoryController extends AbstractLifecycleComponent { private final ThreadPool threadPool; @@ -70,7 +72,7 @@ public class IndexingMemoryBufferController extends AbstractLifecycleComponent activeToInactiveIndexingShards = Lists.newArrayList(); + List inactiveToActiveIndexingShards = Lists.newArrayList(); for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { long time = threadPool.estimatedTimeInMillis(); @@ -145,27 +149,20 @@ public class IndexingMemoryBufferController extends AbstractLifecycleComponent inactiveTime.millis() && indexShard.mergeStats().current() == 0) { - try { - ((InternalIndexShard) indexShard).engine().updateIndexingBufferSize(Engine.INACTIVE_SHARD_INDEXING_BUFFER); - } catch (EngineClosedException e) { - // ignore - continue; - } catch (FlushNotAllowedEngineException e) { - // ignore - continue; - } // inactive for this amount of time, mark it - status.inactive = true; + activeToInactiveIndexingShards.add(indexShard); + status.inactiveIndexing = true; activeInactiveStatusChanges = true; logger.debug("marking shard [{}][{}] as inactive (inactive_time[{}]) indexing wise, setting size to [{}]", indexShard.shardId().index().name(), indexShard.shardId().id(), inactiveTime, Engine.INACTIVE_SHARD_INDEXING_BUFFER); } } } else { - if (status.inactive) { - status.inactive = false; + if (status.inactiveIndexing) { + inactiveToActiveIndexingShards.add(indexShard); + status.inactiveIndexing = false; activeInactiveStatusChanges = true; logger.debug("marking shard [{}][{}] as active indexing wise", indexShard.shardId().index().name(), indexShard.shardId().id()); } @@ -175,6 +172,16 @@ public class IndexingMemoryBufferController extends AbstractLifecycleComponent Date: Sun, 18 Dec 2011 00:59:38 +0200 Subject: [PATCH 049/270] optimize checksum computation to make sure we only work on buffers and no on single bytes --- .../lucene/store/OpenBufferedIndexOutput.java | 151 ++++++++++++++++++ .../org/elasticsearch/index/store/Store.java | 32 ++-- 2 files changed, 161 insertions(+), 22 deletions(-) create mode 100644 src/main/java/org/apache/lucene/store/OpenBufferedIndexOutput.java diff --git a/src/main/java/org/apache/lucene/store/OpenBufferedIndexOutput.java b/src/main/java/org/apache/lucene/store/OpenBufferedIndexOutput.java new file mode 100644 index 00000000000..8a5d1f9b832 --- /dev/null +++ b/src/main/java/org/apache/lucene/store/OpenBufferedIndexOutput.java @@ -0,0 +1,151 @@ +package org.apache.lucene.store; + +import java.io.IOException; + +/** + * Exactly the same as Lucene {@link BufferedIndexOutput} but with the ability to set the buffer size + */ +// LUCENE MONITOR +public abstract class OpenBufferedIndexOutput extends IndexOutput { + + public static final int DEFAULT_BUFFER_SIZE = BufferedIndexOutput.BUFFER_SIZE; + + final int BUFFER_SIZE; + + private final byte[] buffer; + private long bufferStart = 0; // position in file of buffer + private int bufferPosition = 0; // position in buffer + + protected OpenBufferedIndexOutput(int BUFFER_SIZE) { + this.BUFFER_SIZE = BUFFER_SIZE; + this.buffer = new byte[BUFFER_SIZE]; + } + + /** + * Writes a single byte. + * + * @see IndexInput#readByte() + */ + @Override + public void writeByte(byte b) throws IOException { + if (bufferPosition >= BUFFER_SIZE) + flush(); + buffer[bufferPosition++] = b; + } + + /** + * Writes an array of bytes. + * + * @param b the bytes to write + * @param length the number of bytes to write + * @see IndexInput#readBytes(byte[], int, int) + */ + @Override + public void writeBytes(byte[] b, int offset, int length) throws IOException { + int bytesLeft = BUFFER_SIZE - bufferPosition; + // is there enough space in the buffer? + if (bytesLeft >= length) { + // we add the data to the end of the buffer + System.arraycopy(b, offset, buffer, bufferPosition, length); + bufferPosition += length; + // if the buffer is full, flush it + if (BUFFER_SIZE - bufferPosition == 0) + flush(); + } else { + // is data larger then buffer? + if (length > BUFFER_SIZE) { + // we flush the buffer + if (bufferPosition > 0) + flush(); + // and write data at once + flushBuffer(b, offset, length); + bufferStart += length; + } else { + // we fill/flush the buffer (until the input is written) + int pos = 0; // position in the input data + int pieceLength; + while (pos < length) { + pieceLength = (length - pos < bytesLeft) ? length - pos : bytesLeft; + System.arraycopy(b, pos + offset, buffer, bufferPosition, pieceLength); + pos += pieceLength; + bufferPosition += pieceLength; + // if the buffer is full, flush it + bytesLeft = BUFFER_SIZE - bufferPosition; + if (bytesLeft == 0) { + flush(); + bytesLeft = BUFFER_SIZE; + } + } + } + } + } + + /** + * Forces any buffered output to be written. + */ + @Override + public void flush() throws IOException { + flushBuffer(buffer, bufferPosition); + bufferStart += bufferPosition; + bufferPosition = 0; + } + + /** + * Expert: implements buffer write. Writes bytes at the current position in + * the output. + * + * @param b the bytes to write + * @param len the number of bytes to write + */ + private void flushBuffer(byte[] b, int len) throws IOException { + flushBuffer(b, 0, len); + } + + /** + * Expert: implements buffer write. Writes bytes at the current position in + * the output. + * + * @param b the bytes to write + * @param offset the offset in the byte array + * @param len the number of bytes to write + */ + protected abstract void flushBuffer(byte[] b, int offset, int len) throws IOException; + + /** + * Closes this stream to further operations. + */ + @Override + public void close() throws IOException { + flush(); + } + + /** + * Returns the current position in this file, where the next write will + * occur. + * + * @see #seek(long) + */ + @Override + public long getFilePointer() { + return bufferStart + bufferPosition; + } + + /** + * Sets current position in this file, where the next write will occur. + * + * @see #getFilePointer() + */ + @Override + public void seek(long pos) throws IOException { + flush(); + bufferStart = pos; + } + + /** + * The number of bytes in the file. + */ + @Override + public abstract long length() throws IOException; + + +} diff --git a/src/main/java/org/elasticsearch/index/store/Store.java b/src/main/java/org/elasticsearch/index/store/Store.java index 1e5b402518d..bc1236e84dc 100644 --- a/src/main/java/org/elasticsearch/index/store/Store.java +++ b/src/main/java/org/elasticsearch/index/store/Store.java @@ -524,7 +524,7 @@ public class Store extends AbstractIndexShardComponent { } } - class StoreIndexOutput extends IndexOutput { + class StoreIndexOutput extends OpenBufferedIndexOutput { private final StoreFileMetaData metaData; @@ -535,6 +535,9 @@ public class Store extends AbstractIndexShardComponent { private final Checksum digest; StoreIndexOutput(StoreFileMetaData metaData, IndexOutput delegate, String name, boolean computeChecksum) { + // we add 8 to be bigger than the default BufferIndexOutput buffer size so any flush will go directly + // to the output without being copied over to the delegate buffer + super(OpenBufferedIndexOutput.DEFAULT_BUFFER_SIZE + 64); this.metaData = metaData; this.delegate = delegate; this.name = name; @@ -559,6 +562,7 @@ public class Store extends AbstractIndexShardComponent { @Override public void close() throws IOException { + super.close(); delegate.close(); String checksum = null; if (digest != null) { @@ -572,18 +576,10 @@ public class Store extends AbstractIndexShardComponent { } @Override - public void writeByte(byte b) throws IOException { - delegate.writeByte(b); + protected void flushBuffer(byte[] b, int offset, int len) throws IOException { + delegate.writeBytes(b, offset, len); if (digest != null) { - digest.update(b); - } - } - - @Override - public void writeBytes(byte[] b, int offset, int length) throws IOException { - delegate.writeBytes(b, offset, length); - if (digest != null) { - digest.update(b, offset, length); + digest.update(b, offset, len); } } @@ -594,19 +590,16 @@ public class Store extends AbstractIndexShardComponent { @Override public void flush() throws IOException { + super.flush(); delegate.flush(); } - @Override - public long getFilePointer() { - return delegate.getFilePointer(); - } - @Override public void seek(long pos) throws IOException { // seek might be called on files, which means that the checksum is not file checksum // but a checksum of the bytes written to this stream, which is the same for each // type of file in lucene + super.seek(pos); delegate.seek(pos); } @@ -619,10 +612,5 @@ public class Store extends AbstractIndexShardComponent { public void setLength(long length) throws IOException { delegate.setLength(length); } - - @Override - public void writeStringStringMap(Map map) throws IOException { - delegate.writeStringStringMap(map); - } } } From 0328a300eb664c0806b1a6225ec65db675d6bc32 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 18 Dec 2011 01:01:12 +0200 Subject: [PATCH 050/270] move jmeter files under jmeter, no need for jmx --- src/test/resources/jmeter/{jmx => }/index-count.jmx | 0 src/test/resources/jmeter/{jmx => }/index-get.jmx | 0 src/test/resources/jmeter/{jmx => }/index-search.jmx | 0 src/test/resources/jmeter/{jmx => }/index.jmx | 0 src/test/resources/jmeter/{jmx => }/ping-single.jmx | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename src/test/resources/jmeter/{jmx => }/index-count.jmx (100%) rename src/test/resources/jmeter/{jmx => }/index-get.jmx (100%) rename src/test/resources/jmeter/{jmx => }/index-search.jmx (100%) rename src/test/resources/jmeter/{jmx => }/index.jmx (100%) rename src/test/resources/jmeter/{jmx => }/ping-single.jmx (100%) diff --git a/src/test/resources/jmeter/jmx/index-count.jmx b/src/test/resources/jmeter/index-count.jmx similarity index 100% rename from src/test/resources/jmeter/jmx/index-count.jmx rename to src/test/resources/jmeter/index-count.jmx diff --git a/src/test/resources/jmeter/jmx/index-get.jmx b/src/test/resources/jmeter/index-get.jmx similarity index 100% rename from src/test/resources/jmeter/jmx/index-get.jmx rename to src/test/resources/jmeter/index-get.jmx diff --git a/src/test/resources/jmeter/jmx/index-search.jmx b/src/test/resources/jmeter/index-search.jmx similarity index 100% rename from src/test/resources/jmeter/jmx/index-search.jmx rename to src/test/resources/jmeter/index-search.jmx diff --git a/src/test/resources/jmeter/jmx/index.jmx b/src/test/resources/jmeter/index.jmx similarity index 100% rename from src/test/resources/jmeter/jmx/index.jmx rename to src/test/resources/jmeter/index.jmx diff --git a/src/test/resources/jmeter/jmx/ping-single.jmx b/src/test/resources/jmeter/ping-single.jmx similarity index 100% rename from src/test/resources/jmeter/jmx/ping-single.jmx rename to src/test/resources/jmeter/ping-single.jmx From 33db88232d6a97236a155c3ebb2fe208db63b632 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 19 Dec 2011 11:43:50 +0200 Subject: [PATCH 051/270] move to 0.18.7 --- src/main/java/org/elasticsearch/Version.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/elasticsearch/Version.java b/src/main/java/org/elasticsearch/Version.java index 26481821c69..ba75c07541b 100644 --- a/src/main/java/org/elasticsearch/Version.java +++ b/src/main/java/org/elasticsearch/Version.java @@ -48,6 +48,8 @@ public class Version { public static final Version V_0_18_5 = new Version(V_0_18_5_ID, false); public static final int V_0_18_6_ID = /*00*/180699; public static final Version V_0_18_6 = new Version(V_0_18_6_ID, false); + public static final int V_0_18_7_ID = /*00*/180799; + public static final Version V_0_18_7 = new Version(V_0_18_7_ID, false); public static final int V_0_19_0_ID = /*00*/190099; public static final Version V_0_19_0 = new Version(V_0_19_0_ID, true); @@ -74,6 +76,9 @@ public class Version { return V_0_18_5; case V_0_18_6_ID: return V_0_18_6; + case V_0_18_7_ID: + return V_0_18_7; + case V_0_19_0_ID: return V_0_19_0; default: From 41b5c3d562bfa449d922e743f1279c7d6d634821 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 19 Dec 2011 13:51:33 +0200 Subject: [PATCH 052/270] wait for yellow state in test --- .../integration/search/scriptfield/ScriptFieldSearchTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/org/elasticsearch/test/integration/search/scriptfield/ScriptFieldSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/scriptfield/ScriptFieldSearchTests.java index 5cfb51bc4f4..0e75bec50a8 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/scriptfield/ScriptFieldSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/scriptfield/ScriptFieldSearchTests.java @@ -145,6 +145,7 @@ public class ScriptFieldSearchTests extends AbstractNodesTests { // its ok } client.admin().indices().prepareCreate("test").execute().actionGet(); + client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); client.prepareIndex("test", "type1", "1") .setSource(jsonBuilder().startObject() .startObject("obj1").field("test", "something").endObject() From e827c56be4a48499729f6bc50fec8262da79305a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 19 Dec 2011 13:51:50 +0200 Subject: [PATCH 053/270] move to 1.9.3 jackson --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2aa96354f6d..c2d40dd6936 100644 --- a/pom.xml +++ b/pom.xml @@ -146,14 +146,14 @@ org.codehaus.jackson jackson-core-asl - 1.9.2 + 1.9.3 compile org.codehaus.jackson jackson-smile - 1.9.2 + 1.9.3 compile From 1f250ede501b6d41e79c6682bc42f635102ed752 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 19 Dec 2011 15:51:05 +0200 Subject: [PATCH 054/270] better support for "embedded" token types (binaries) when parsing --- .../org/elasticsearch/common/xcontent/XContentHelper.java | 2 ++ .../common/xcontent/support/XContentMapConverter.java | 2 ++ .../org/elasticsearch/index/mapper/object/ObjectMapper.java | 6 ++++++ 3 files changed, 10 insertions(+) diff --git a/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 966d966c98a..d48520791a7 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -187,6 +187,8 @@ public class XContentHelper { case VALUE_NULL: generator.writeNull(); break; + case VALUE_EMBEDDED_OBJECT: + generator.writeBinary(parser.binaryValue()); } } diff --git a/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapConverter.java b/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapConverter.java index af2787d6a57..afaaea02030 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapConverter.java +++ b/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapConverter.java @@ -107,6 +107,8 @@ public class XContentMapConverter { return readMap(parser, mapFactory); } else if (t == XContentParser.Token.START_ARRAY) { return readList(parser, mapFactory, t); + } else if (t == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + return parser.binaryValue(); } return null; } diff --git a/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java b/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java index 53728501120..c92a1b8b08d 100644 --- a/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java @@ -736,6 +736,12 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll { builder = booleanField(currentFieldName); } mapper = builder.build(builderContext); + } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "binary"); + if (builder == null) { + builder = binaryField(currentFieldName); + } + mapper = builder.build(builderContext); } else { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, null); if (builder != null) { From dd6c076454406249518e622ab4111c6f373497c7 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 20 Dec 2011 12:03:28 +0200 Subject: [PATCH 055/270] simplify and improve scaling/blocking thread pools --- .../util/concurrent/DynamicExecutors.java | 126 ---- .../concurrent/DynamicThreadPoolExecutor.java | 166 ----- .../common/util/concurrent/EsExecutors.java | 105 ++- .../TransferThreadPoolExecutor.java | 635 ------------------ .../zen/ping/unicast/UnicastZenPing.java | 3 +- .../elasticsearch/gateway/fs/FsGateway.java | 5 +- .../indices/recovery/RecoverySettings.java | 5 +- .../elasticsearch/threadpool/ThreadPool.java | 5 +- .../concurrent/BlockingThreadPoolTest.java | 95 --- ...eadPoolTest.java => EsExecutorsTests.java} | 69 +- 10 files changed, 146 insertions(+), 1068 deletions(-) delete mode 100644 src/main/java/org/elasticsearch/common/util/concurrent/DynamicExecutors.java delete mode 100644 src/main/java/org/elasticsearch/common/util/concurrent/DynamicThreadPoolExecutor.java delete mode 100644 src/main/java/org/elasticsearch/common/util/concurrent/TransferThreadPoolExecutor.java delete mode 100644 src/test/java/org/elasticsearch/test/unit/common/util/concurrent/BlockingThreadPoolTest.java rename src/test/java/org/elasticsearch/test/unit/common/util/concurrent/{ScalingThreadPoolTest.java => EsExecutorsTests.java} (66%) diff --git a/src/main/java/org/elasticsearch/common/util/concurrent/DynamicExecutors.java b/src/main/java/org/elasticsearch/common/util/concurrent/DynamicExecutors.java deleted file mode 100644 index 017d0252685..00000000000 --- a/src/main/java/org/elasticsearch/common/util/concurrent/DynamicExecutors.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.util.concurrent; - -import java.util.concurrent.*; - -/** - * - */ -public class DynamicExecutors { - - /** - * Creates a thread pool that creates new threads as needed, but will reuse - * previously constructed threads when they are available. Calls to - * execute will reuse previously constructed threads if - * available. If no existing thread is available, a new thread will be - * created and added to the pool. No more than max threads will - * be created. Threads that have not been used for a keepAlive - * timeout are terminated and removed from the cache. Thus, a pool that - * remains idle for long enough will not consume any resources other than - * the min specified. - * - * @param min the number of threads to keep in the pool, even if they are - * idle. - * @param max the maximum number of threads to allow in the pool. - * @param keepAliveTime when the number of threads is greater than the min, - * this is the maximum time that excess idle threads will wait - * for new tasks before terminating (in milliseconds). - * @return the newly created thread pool - */ - public static ExecutorService newScalingThreadPool(int min, int max, long keepAliveTime) { - return newScalingThreadPool(min, max, keepAliveTime, Executors.defaultThreadFactory()); - } - - /** - * Creates a thread pool, same as in - * {@link #newScalingThreadPool(int, int, long)}, using the provided - * ThreadFactory to create new threads when needed. - * - * @param min the number of threads to keep in the pool, even if they are - * idle. - * @param max the maximum number of threads to allow in the pool. - * @param keepAliveTime when the number of threads is greater than the min, - * this is the maximum time that excess idle threads will wait - * for new tasks before terminating (in milliseconds). - * @param threadFactory the factory to use when creating new threads. - * @return the newly created thread pool - */ - public static ExecutorService newScalingThreadPool(int min, int max, long keepAliveTime, ThreadFactory threadFactory) { - DynamicThreadPoolExecutor.DynamicQueue queue = new DynamicThreadPoolExecutor.DynamicQueue(); - ThreadPoolExecutor executor = new DynamicThreadPoolExecutor(min, max, keepAliveTime, TimeUnit.MILLISECONDS, queue, threadFactory); - executor.setRejectedExecutionHandler(new DynamicThreadPoolExecutor.ForceQueuePolicy()); - queue.setThreadPoolExecutor(executor); - return executor; - } - - /** - * Creates a thread pool similar to that constructed by - * {@link #newScalingThreadPool(int, int, long)}, but blocks the call to - * execute if the queue has reached it's capacity, and all - * max threads are busy handling requests. - *

- * If the wait time of this queue has elapsed, a - * {@link RejectedExecutionException} will be thrown. - * - * @param min the number of threads to keep in the pool, even if they are - * idle. - * @param max the maximum number of threads to allow in the pool. - * @param keepAliveTime when the number of threads is greater than the min, - * this is the maximum time that excess idle threads will wait - * for new tasks before terminating (in milliseconds). - * @param capacity the fixed capacity of the underlying queue (resembles - * backlog). - * @param waitTime the wait time (in milliseconds) for space to become - * available in the queue. - * @return the newly created thread pool - */ - public static ExecutorService newBlockingThreadPool(int min, int max, long keepAliveTime, int capacity, long waitTime) { - return newBlockingThreadPool(min, max, keepAliveTime, capacity, waitTime, Executors.defaultThreadFactory()); - } - - /** - * Creates a thread pool, same as in - * {@link #newBlockingThreadPool(int, int, long, int, long)}, using the - * provided ThreadFactory to create new threads when needed. - * - * @param min the number of threads to keep in the pool, even if they are - * idle. - * @param max the maximum number of threads to allow in the pool. - * @param keepAliveTime when the number of threads is greater than the min, - * this is the maximum time that excess idle threads will wait - * for new tasks before terminating (in milliseconds). - * @param capacity the fixed capacity of the underlying queue (resembles - * backlog). - * @param waitTime the wait time (in milliseconds) for space to become - * available in the queue. - * @param threadFactory the factory to use when creating new threads. - * @return the newly created thread pool - */ - public static ExecutorService newBlockingThreadPool(int min, int max, - long keepAliveTime, int capacity, long waitTime, - ThreadFactory threadFactory) { - DynamicThreadPoolExecutor.DynamicQueue queue = new DynamicThreadPoolExecutor.DynamicQueue(capacity); - ThreadPoolExecutor executor = new DynamicThreadPoolExecutor(min, max, keepAliveTime, TimeUnit.MILLISECONDS, queue, threadFactory); - executor.setRejectedExecutionHandler(new DynamicThreadPoolExecutor.TimedBlockingPolicy(waitTime)); - queue.setThreadPoolExecutor(executor); - return executor; - } -} diff --git a/src/main/java/org/elasticsearch/common/util/concurrent/DynamicThreadPoolExecutor.java b/src/main/java/org/elasticsearch/common/util/concurrent/DynamicThreadPoolExecutor.java deleted file mode 100644 index 69f4728a68f..00000000000 --- a/src/main/java/org/elasticsearch/common/util/concurrent/DynamicThreadPoolExecutor.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.util.concurrent; - -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * An {@link ExecutorService} that executes each submitted task using one of - * possibly several pooled threads, normally configured using - * {@link DynamicExecutors} factory methods. - * - * - */ -public class DynamicThreadPoolExecutor extends ThreadPoolExecutor { - /** - * number of threads that are actively executing tasks - */ - private final AtomicInteger activeCount = new AtomicInteger(); - - public DynamicThreadPoolExecutor(int corePoolSize, int maximumPoolSize, - long keepAliveTime, TimeUnit unit, BlockingQueue workQueue, - ThreadFactory threadFactory) { - super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); - } - - @Override - public int getActiveCount() { - return activeCount.get(); - } - - @Override - protected void beforeExecute(Thread t, Runnable r) { - activeCount.incrementAndGet(); - } - - @Override - protected void afterExecute(Runnable r, Throwable t) { - activeCount.decrementAndGet(); - } - - /** - * Much like a {@link SynchronousQueue} which acts as a rendezvous channel. It - * is well suited for handoff designs, in which a tasks is only queued if there - * is an available thread to pick it up. - *

- * This queue is correlated with a thread-pool, and allows insertions to the - * queue only if there is a free thread that can poll this task. Otherwise, the - * task is rejected and the decision is left up to one of the - * {@link RejectedExecutionHandler} policies: - *

    - *
  1. {@link ForceQueuePolicy} - forces the queue to accept the rejected task.
    2. - *
  2. {@link TimedBlockingPolicy} - waits for a given time for the task to be - * executed.
    3. - *
- * - * - */ - public static class DynamicQueue extends LinkedBlockingQueue { - private static final long serialVersionUID = 1L; - - /** - * The executor this Queue belongs to - */ - private transient ThreadPoolExecutor executor; - - /** - * Creates a DynamicQueue with a capacity of - * {@link Integer#MAX_VALUE}. - */ - public DynamicQueue() { - super(); - } - - /** - * Creates a DynamicQueue with the given (fixed) capacity. - * - * @param capacity the capacity of this queue. - */ - public DynamicQueue(int capacity) { - super(capacity); - } - - /** - * Sets the executor this queue belongs to. - */ - public void setThreadPoolExecutor(ThreadPoolExecutor executor) { - this.executor = executor; - } - - /** - * Inserts the specified element at the tail of this queue if there is at - * least one available thread to run the current task. If all pool threads - * are actively busy, it rejects the offer. - * - * @param o the element to add. - * @return true if it was possible to add the element to this - * queue, else false - * @see ThreadPoolExecutor#execute(Runnable) - */ - @Override - public boolean offer(E o) { - int allWorkingThreads = executor.getActiveCount() + super.size(); - return allWorkingThreads < executor.getPoolSize() && super.offer(o); - } - } - - /** - * A handler for rejected tasks that adds the specified element to this queue, - * waiting if necessary for space to become available. - */ - public static class ForceQueuePolicy implements RejectedExecutionHandler { - public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { - try { - executor.getQueue().put(r); - } catch (InterruptedException e) { - //should never happen since we never wait - throw new RejectedExecutionException(e); - } - } - } - - /** - * A handler for rejected tasks that inserts the specified element into this - * queue, waiting if necessary up to the specified wait time for space to become - * available. - */ - public static class TimedBlockingPolicy implements RejectedExecutionHandler { - private final long waitTime; - - /** - * @param waitTime wait time in milliseconds for space to become available. - */ - public TimedBlockingPolicy(long waitTime) { - this.waitTime = waitTime; - } - - public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { - try { - boolean successful = executor.getQueue().offer(r, waitTime, TimeUnit.MILLISECONDS); - if (!successful) - throw new RejectedExecutionException("Rejected execution after waiting " - + waitTime + " ms for task [" + r.getClass() + "] to be executed."); - } catch (InterruptedException e) { - throw new RejectedExecutionException(e); - } - } - } -} diff --git a/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java index 91399716063..6c32b4832ca 100644 --- a/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java +++ b/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java @@ -19,8 +19,8 @@ package org.elasticsearch.common.util.concurrent; +import jsr166y.LinkedTransferQueue; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; import java.util.concurrent.*; @@ -29,11 +29,24 @@ import java.util.concurrent.*; */ public class EsExecutors { - public static ExecutorService newCachedThreadPool(TimeValue keepAlive, ThreadFactory threadFactory) { - return new ThreadPoolExecutor(0, Integer.MAX_VALUE, - keepAlive.millis(), TimeUnit.MILLISECONDS, - new SynchronousQueue(), - threadFactory); + public static ThreadPoolExecutor newScalingExecutorService(int min, int max, long keepAliveTime, TimeUnit unit, + ThreadFactory threadFactory) { + ExecutorScalingQueue queue = new ExecutorScalingQueue(); + // we force the execution, since we might run into concurrency issues in offer for ScalingBlockingQueue + ThreadPoolExecutor executor = new ThreadPoolExecutor(min, max, keepAliveTime, unit, queue, threadFactory, + new ForceQueuePolicy()); + queue.executor = executor; + return executor; + } + + public static ThreadPoolExecutor newBlockingExecutorService(int min, int max, long keepAliveTime, TimeUnit unit, + ThreadFactory threadFactory, int capacity, + long waitTime, TimeUnit waitTimeUnit) { + ExecutorBlockingQueue queue = new ExecutorBlockingQueue(capacity); + ThreadPoolExecutor executor = new ThreadPoolExecutor(min, max, keepAliveTime, unit, queue, threadFactory, + new TimedBlockingPolicy(waitTimeUnit.toMillis(waitTime))); + queue.executor = executor; + return executor; } public static String threadName(Settings settings, String namePrefix) { @@ -88,4 +101,84 @@ public class EsExecutors { */ private EsExecutors() { } + + static class ExecutorScalingQueue extends LinkedTransferQueue { + + ThreadPoolExecutor executor; + + public ExecutorScalingQueue() { + } + + @Override + public boolean offer(E e) { + int left = executor.getMaximumPoolSize() - executor.getCorePoolSize(); + if (!tryTransfer(e)) { + if (left > 0) { + return false; + } else { + return super.offer(e); + } + } else { + return true; + } + } + } + + static class ExecutorBlockingQueue extends ArrayBlockingQueue { + + ThreadPoolExecutor executor; + + ExecutorBlockingQueue(int capacity) { + super(capacity); + } + + @Override + public boolean offer(E o) { + int allWorkingThreads = executor.getActiveCount() + super.size(); + return allWorkingThreads < executor.getPoolSize() && super.offer(o); + } + } + + + /** + * A handler for rejected tasks that adds the specified element to this queue, + * waiting if necessary for space to become available. + */ + static class ForceQueuePolicy implements RejectedExecutionHandler { + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + try { + executor.getQueue().put(r); + } catch (InterruptedException e) { + //should never happen since we never wait + throw new RejectedExecutionException(e); + } + } + } + + /** + * A handler for rejected tasks that inserts the specified element into this + * queue, waiting if necessary up to the specified wait time for space to become + * available. + */ + static class TimedBlockingPolicy implements RejectedExecutionHandler { + private final long waitTime; + + /** + * @param waitTime wait time in milliseconds for space to become available. + */ + public TimedBlockingPolicy(long waitTime) { + this.waitTime = waitTime; + } + + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + try { + boolean successful = executor.getQueue().offer(r, waitTime, TimeUnit.MILLISECONDS); + if (!successful) + throw new RejectedExecutionException("Rejected execution after waiting " + + waitTime + " ms for task [" + r.getClass() + "] to be executed."); + } catch (InterruptedException e) { + throw new RejectedExecutionException(e); + } + } + } } diff --git a/src/main/java/org/elasticsearch/common/util/concurrent/TransferThreadPoolExecutor.java b/src/main/java/org/elasticsearch/common/util/concurrent/TransferThreadPoolExecutor.java deleted file mode 100644 index a415588e71b..00000000000 --- a/src/main/java/org/elasticsearch/common/util/concurrent/TransferThreadPoolExecutor.java +++ /dev/null @@ -1,635 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.util.concurrent; - -import jsr166y.LinkedTransferQueue; -import jsr166y.TransferQueue; - -import java.util.*; -import java.util.concurrent.AbstractExecutorService; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.ReentrantLock; - -/** - * A thread pool based on {@link jsr166y.TransferQueue}. - *

- *

Limited compared to ExecutorServer in what it does, but focused on speed. - * - * - */ -public class TransferThreadPoolExecutor extends AbstractExecutorService { - - private final TransferQueue workQueue = new LinkedTransferQueue(); - - private final AtomicInteger queueSize = new AtomicInteger(); - - /** - * Lock held on updates to poolSize, corePoolSize, - * maximumPoolSize, runState, and workers set. - */ - private final ReentrantLock mainLock = new ReentrantLock(); - - /** - * Wait condition to support awaitTermination - */ - private final Condition termination = mainLock.newCondition(); - - /** - * Set containing all worker threads in pool. Accessed only when - * holding mainLock. - */ - private final HashSet workers = new HashSet(); - - - /** - * Factory for new threads. All threads are created using this - * factory (via method addThread). All callers must be prepared - * for addThread to fail by returning null, which may reflect a - * system or user's policy limiting the number of threads. Even - * though it is not treated as an error, failure to create threads - * may result in new tasks being rejected or existing ones - * remaining stuck in the queue. On the other hand, no special - * precautions exist to handle OutOfMemoryErrors that might be - * thrown while trying to create threads, since there is generally - * no recourse from within this class. - */ - private final ThreadFactory threadFactory; - - /** - * runState provides the main lifecyle control, taking on values: - *

- * RUNNING: Accept new tasks and process queued tasks - * SHUTDOWN: Don't accept new tasks, but process queued tasks - * STOP: Don't accept new tasks, don't process queued tasks, - * and interrupt in-progress tasks - * TERMINATED: Same as STOP, plus all threads have terminated - *

- * The numerical order among these values matters, to allow - * ordered comparisons. The runState monotonically increases over - * time, but need not hit each state. The transitions are: - *

- * RUNNING -> SHUTDOWN - * On invocation of shutdown(), perhaps implicitly in finalize() - * (RUNNING or SHUTDOWN) -> STOP - * On invocation of shutdownNow() - * SHUTDOWN -> TERMINATED - * When both queue and pool are empty - * STOP -> TERMINATED - * When pool is empty - */ - volatile int runState; - static final int RUNNING = 0; - static final int SHUTDOWN = 1; - static final int STOP = 2; - static final int TERMINATED = 3; - - - private final boolean blocking; - - private final int blockingCapacity; - - private final long blockingTime; - - /** - * Core pool size, updated only while holding mainLock, but - * volatile to allow concurrent readability even during updates. - */ - private final int corePoolSize; - - /** - * Maximum pool size, updated only while holding mainLock but - * volatile to allow concurrent readability even during updates. - */ - private final int maximumPoolSize; - - /** - * Timeout in nanoseconds for idle threads waiting for work. - * Threads use this timeout when there are more than corePoolSize - * present or if allowCoreThreadTimeOut. Otherwise they wait - * forever for new work. - */ - private final long keepAliveTime; - - /** - * Current pool size, updated only while holding mainLock but - * volatile to allow concurrent readability even during updates. - */ - private final AtomicInteger poolSize = new AtomicInteger(); - - public static TransferThreadPoolExecutor newScalingExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, ThreadFactory threadFactory) { - return new TransferThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, false, 0, TimeUnit.NANOSECONDS, 0, threadFactory); - } - - public static TransferThreadPoolExecutor newBlockingExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, - long blockingTime, TimeUnit blockingUnit, int blockingCapacity, - ThreadFactory threadFactory) { - return new TransferThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, true, blockingTime, blockingUnit, blockingCapacity, threadFactory); - } - - private TransferThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, - boolean blocking, long blockingTime, TimeUnit blockingUnit, int blockingCapacity, - ThreadFactory threadFactory) { - this.blocking = blocking; - this.blockingTime = blockingUnit.toNanos(blockingTime); - this.blockingCapacity = blockingCapacity; - this.corePoolSize = corePoolSize; - this.maximumPoolSize = maximumPoolSize; - this.keepAliveTime = unit.toNanos(keepAliveTime); - this.threadFactory = threadFactory; - - for (int i = 0; i < corePoolSize; i++) { - Thread t = addWorker(); - if (t != null) { - poolSize.incrementAndGet(); - t.start(); - } - } - } - - - @Override - public void execute(Runnable command) { - if (blocking) { - executeBlocking(command); - } else { - executeNonBlocking(command); - } - } - - private void executeNonBlocking(Runnable command) { - // note, there might be starvation of some commands that were added to the queue, - // while others are being transferred directly - queueSize.getAndIncrement(); - boolean succeeded = workQueue.tryTransfer(command); - if (succeeded) { - return; - } - int currentPoolSize = poolSize.get(); - if (currentPoolSize < maximumPoolSize) { - // if we manage to add a worker, add it, and tryTransfer again - if (poolSize.compareAndSet(currentPoolSize, currentPoolSize + 1)) { - Thread t = addWorker(); - if (t == null) { - poolSize.decrementAndGet(); - workQueue.add(command); - } else { - t.start(); - succeeded = workQueue.tryTransfer(command); - if (!succeeded) { - workQueue.add(command); - } - } - } else { - succeeded = workQueue.tryTransfer(command); - if (!succeeded) { - workQueue.add(command); - } - } - } else { - workQueue.add(command); - } - } - - private void executeBlocking(Runnable command) { - int currentCapacity = queueSize.getAndIncrement(); - boolean succeeded = workQueue.tryTransfer(command); - if (succeeded) { - return; - } - int currentPoolSize = poolSize.get(); - if (currentPoolSize < maximumPoolSize) { - // if we manage to add a worker, add it, and tryTransfer again - if (poolSize.compareAndSet(currentPoolSize, currentPoolSize + 1)) { - Thread t = addWorker(); - if (t == null) { - poolSize.decrementAndGet(); - workQueue.add(command); - } else { - t.start(); - succeeded = workQueue.tryTransfer(command); - if (!succeeded) { - transferOrAddBlocking(command, currentCapacity); - } - } - } else { - succeeded = workQueue.tryTransfer(command); - if (!succeeded) { - transferOrAddBlocking(command, currentCapacity); - } - } - } else { - transferOrAddBlocking(command, currentCapacity); - } - } - - private void transferOrAddBlocking(Runnable command, int currentCapacity) { - if (currentCapacity < blockingCapacity) { - workQueue.add(command); - } else { - boolean succeeded; - try { - succeeded = workQueue.tryTransfer(command, blockingTime, TimeUnit.NANOSECONDS); - if (!succeeded) { - throw new RejectedExecutionException("Rejected execution after waiting " - + TimeUnit.NANOSECONDS.toSeconds(blockingTime) + "s for task [" + command.getClass() + "] to be executed."); - } - } catch (InterruptedException e) { - throw new RejectedExecutionException(e); - } - } - } - - @Override - public void shutdown() { - final ReentrantLock mainLock = this.mainLock; - mainLock.lock(); - try { - int state = runState; - if (state < SHUTDOWN) - runState = SHUTDOWN; - - try { - for (Worker w : workers) { - w.interruptIfIdle(); - } - } catch (SecurityException se) { // Try to back out - runState = state; - // tryTerminate() here would be a no-op - throw se; - } - - tryTerminate(); // Terminate now if pool and queue empty - } finally { - mainLock.unlock(); - } - } - - @Override - public List shutdownNow() { - final ReentrantLock mainLock = this.mainLock; - mainLock.lock(); - try { - int state = runState; - if (state < STOP) - runState = STOP; - - try { - for (Worker w : workers) { - w.interruptNow(); - } - } catch (SecurityException se) { // Try to back out - runState = state; - // tryTerminate() here would be a no-op - throw se; - } - - List tasks = drainQueue(); - tryTerminate(); // Terminate now if pool and queue empty - return tasks; - } finally { - mainLock.unlock(); - } - } - - @Override - public boolean isShutdown() { - return runState != RUNNING; - } - - @Override - public boolean isTerminated() { - return runState == TERMINATED; - } - - @Override - public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { - long nanos = unit.toNanos(timeout); - final ReentrantLock mainLock = this.mainLock; - mainLock.lock(); - try { - for (; ; ) { - if (runState == TERMINATED) - return true; - if (nanos <= 0) - return false; - nanos = termination.awaitNanos(nanos); - } - } finally { - mainLock.unlock(); - } - } - - /** - * Returns the current number of threads in the pool. - * - * @return the number of threads - */ - public int getPoolSize() { - return poolSize.get(); - } - - /** - * Returns the approximate number of threads that are actively - * executing tasks. - * - * @return the number of threads - */ - public int getActiveCount() { - final ReentrantLock mainLock = this.mainLock; - mainLock.lock(); - try { - int n = 0; - for (Worker w : workers) { - if (w.isActive()) - ++n; - } - return n; - } finally { - mainLock.unlock(); - } - } - - public int getCorePoolSize() { - return corePoolSize; - } - - public int getMaximumPoolSize() { - return maximumPoolSize; - } - - public int getQueueSize() { - return queueSize.get(); - } - - private final class Worker implements Runnable { - /** - * The runLock is acquired and released surrounding each task - * execution. It mainly protects against interrupts that are - * intended to cancel the worker thread from instead - * interrupting the task being run. - */ - private final ReentrantLock runLock = new ReentrantLock(); - - /** - * Thread this worker is running in. Acts as a final field, - * but cannot be set until thread is created. - */ - Thread thread; - - Worker() { - } - - boolean isActive() { - return runLock.isLocked(); - } - - /** - * Interrupts thread if not running a task. - */ - void interruptIfIdle() { - final ReentrantLock runLock = this.runLock; - if (runLock.tryLock()) { - try { - if (thread != Thread.currentThread()) - thread.interrupt(); - } finally { - runLock.unlock(); - } - } - } - - /** - * Interrupts thread even if running a task. - */ - void interruptNow() { - thread.interrupt(); - } - - /** - * Runs a single task between before/after methods. - */ - private void runTask(Runnable task) { - final ReentrantLock runLock = this.runLock; - runLock.lock(); - try { - /* - * Ensure that unless pool is stopping, this thread - * does not have its interrupt set. This requires a - * double-check of state in case the interrupt was - * cleared concurrently with a shutdownNow -- if so, - * the interrupt is re-enabled. - */ - if (runState < STOP && Thread.interrupted() && runState >= STOP) - thread.interrupt(); - - task.run(); - } finally { - runLock.unlock(); - } - } - - /** - * Main run loop - */ - public void run() { - try { - Runnable task; - while ((task = getTask()) != null) { - runTask(task); - } - } finally { - workerDone(this); - } - } - } - - - Runnable getTask() { - for (; ; ) { - try { - int state = runState; - if (state > SHUTDOWN) - return null; - Runnable r; - if (state == SHUTDOWN) // Help drain queue - r = workQueue.poll(); - else if (poolSize.get() > corePoolSize) - r = workQueue.poll(keepAliveTime, TimeUnit.NANOSECONDS); - else - r = workQueue.take(); - if (r != null) { - queueSize.decrementAndGet(); - return r; - } - if (workerCanExit()) { - if (runState >= SHUTDOWN) // Wake up others - interruptIdleWorkers(); - return null; - } - // Else retry - } catch (InterruptedException ie) { - // On interruption, re-check runState - } - } - } - - /** - * Check whether a worker thread that fails to get a task can - * exit. We allow a worker thread to die if the pool is stopping, - * or the queue is empty, or there is at least one thread to - * handle possibly non-empty queue, even if core timeouts are - * allowed. - */ - private boolean workerCanExit() { - final ReentrantLock mainLock = this.mainLock; - mainLock.lock(); - boolean canExit; - try { - canExit = runState >= STOP || (queueSize.get() == 0 && (runState >= SHUTDOWN || poolSize.get() > corePoolSize)); - } finally { - mainLock.unlock(); - } - return canExit; - } - - /** - * Wakes up all threads that might be waiting for tasks so they - * can check for termination. Note: this method is also called by - * ScheduledThreadPoolExecutor. - */ - void interruptIdleWorkers() { - final ReentrantLock mainLock = this.mainLock; - mainLock.lock(); - try { - for (Worker w : workers) - w.interruptIfIdle(); - } finally { - mainLock.unlock(); - } - } - - /** - * Performs bookkeeping for an exiting worker thread. - * - * @param w the worker - */ - void workerDone(Worker w) { - final ReentrantLock mainLock = this.mainLock; - mainLock.lock(); - try { - workers.remove(w); - if (poolSize.decrementAndGet() == 0) - tryTerminate(); - } finally { - mainLock.unlock(); - } - } - - /** - * Transitions to TERMINATED state if either (SHUTDOWN and pool - * and queue empty) or (STOP and pool empty), otherwise unless - * stopped, ensuring that there is at least one live thread to - * handle queued tasks. - *

- * This method is called from the three places in which - * termination can occur: in workerDone on exit of the last thread - * after pool has been shut down, or directly within calls to - * shutdown or shutdownNow, if there are no live threads. - */ - private void tryTerminate() { - if (poolSize.get() == 0) { - int state = runState; - if (state < STOP && queueSize.get() > 0) { - state = RUNNING; // disable termination check below - Thread t = addThread(); - poolSize.incrementAndGet(); - if (t != null) - t.start(); - } - if (state == STOP || state == SHUTDOWN) { - runState = TERMINATED; - termination.signalAll(); - } - } - } - - /** - * Creates and returns a new thread running firstTask as its first - * task. Executed under mainLock. - */ - private Thread addWorker() { - final ReentrantLock mainLock = this.mainLock; - mainLock.lock(); - try { - return addThread(); - } finally { - mainLock.unlock(); - } - } - - /** - * Creates and returns a new thread running firstTask as its first - * task. Call only while holding mainLock. - */ - private Thread addThread() { - Worker w = new Worker(); - Thread t = threadFactory.newThread(w); - if (t != null) { - w.thread = t; - workers.add(w); - } - return t; - } - - /** - * Drains the task queue into a new list. Used by shutdownNow. - * Call only while holding main lock. - */ - private List drainQueue() { - List taskList = new ArrayList(); - workQueue.drainTo(taskList); - queueSize.getAndAdd(taskList.size() * -1); - /* - * If the queue is a DelayQueue or any other kind of queue - * for which poll or drainTo may fail to remove some elements, - * we need to manually traverse and remove remaining tasks. - * To guarantee atomicity wrt other threads using this queue, - * we need to create a new iterator for each element removed. - */ - while (!workQueue.isEmpty()) { - Iterator it = workQueue.iterator(); - try { - if (it.hasNext()) { - Runnable r = it.next(); - if (workQueue.remove(r)) { - taskList.add(r); - queueSize.decrementAndGet(); - } - } - } catch (ConcurrentModificationException ignore) { - } - } - return taskList; - } -} diff --git a/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java b/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java index e1acde6b15a..7a514bcfb2d 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java @@ -35,7 +35,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.common.util.concurrent.DynamicExecutors; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.discovery.zen.DiscoveryNodesProvider; import org.elasticsearch.discovery.zen.ping.ZenPing; @@ -209,7 +208,7 @@ public class UnicastZenPing extends AbstractLifecycleComponent implemen public Executor executor() { if (executor == null) { ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(settings, "[unicast_connect]"); - executor = DynamicExecutors.newScalingThreadPool(1, concurrentConnects, 60000, threadFactory); + executor = EsExecutors.newScalingExecutorService(0, concurrentConnects, 60, TimeUnit.SECONDS, threadFactory); } return executor; } diff --git a/src/main/java/org/elasticsearch/gateway/fs/FsGateway.java b/src/main/java/org/elasticsearch/gateway/fs/FsGateway.java index c8210c215fc..8bcfb26430d 100644 --- a/src/main/java/org/elasticsearch/gateway/fs/FsGateway.java +++ b/src/main/java/org/elasticsearch/gateway/fs/FsGateway.java @@ -26,8 +26,6 @@ import org.elasticsearch.common.blobstore.fs.FsBlobStore; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.DynamicExecutors; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.env.Environment; import org.elasticsearch.gateway.blobstore.BlobStoreGateway; @@ -37,6 +35,7 @@ import org.elasticsearch.threadpool.ThreadPool; import java.io.File; import java.io.IOException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; /** * @@ -60,7 +59,7 @@ public class FsGateway extends BlobStoreGateway { } int concurrentStreams = componentSettings.getAsInt("concurrent_streams", 5); - this.concurrentStreamPool = DynamicExecutors.newScalingThreadPool(1, concurrentStreams, TimeValue.timeValueSeconds(5).millis(), EsExecutors.daemonThreadFactory(settings, "[fs_stream]")); + this.concurrentStreamPool = EsExecutors.newScalingExecutorService(1, concurrentStreams, 60, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory(settings, "[fs_stream]")); initialize(new FsBlobStore(componentSettings, concurrentStreamPool, gatewayFile), clusterName, null); } diff --git a/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java b/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java index 737d5e6b4f8..715dca6e8e5 100644 --- a/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java +++ b/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java @@ -27,12 +27,11 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.DynamicExecutors; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.node.settings.NodeSettingsService; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** */ @@ -69,7 +68,7 @@ public class RecoverySettings extends AbstractComponent { this.compress = componentSettings.getAsBoolean("compress", true); this.concurrentStreams = componentSettings.getAsInt("concurrent_streams", settings.getAsInt("index.shard.recovery.concurrent_streams", 5)); - this.concurrentStreamPool = (ThreadPoolExecutor) DynamicExecutors.newScalingThreadPool(1, concurrentStreams, TimeValue.timeValueSeconds(5).millis(), EsExecutors.daemonThreadFactory(settings, "[recovery_stream]")); + this.concurrentStreamPool = EsExecutors.newScalingExecutorService(0, concurrentStreams, 60, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory(settings, "[recovery_stream]")); this.maxSizePerSec = componentSettings.getAsBytesSize("max_size_per_sec", new ByteSizeValue(0)); if (maxSizePerSec.bytes() <= 0) { diff --git a/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index feb065d9116..e0cd7ae3c34 100644 --- a/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.SizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.DynamicExecutors; import org.elasticsearch.common.util.concurrent.EsExecutors; import java.util.Map; @@ -194,7 +193,7 @@ public class ThreadPool extends AbstractComponent { int min = settings.getAsInt("min", defaultSettings.getAsInt("min", 1)); int size = settings.getAsInt("size", defaultSettings.getAsInt("size", Runtime.getRuntime().availableProcessors() * 5)); logger.debug("creating thread_pool [{}], type [{}], min [{}], size [{}], keep_alive [{}]", name, type, min, size, keepAlive); - return DynamicExecutors.newScalingThreadPool(min, size, keepAlive.millis(), threadFactory); + return EsExecutors.newScalingExecutorService(min, size, keepAlive.millis(), TimeUnit.MILLISECONDS, threadFactory); } else if ("blocking".equals(type)) { TimeValue keepAlive = settings.getAsTime("keep_alive", defaultSettings.getAsTime("keep_alive", timeValueMinutes(5))); int min = settings.getAsInt("min", defaultSettings.getAsInt("min", 1)); @@ -202,7 +201,7 @@ public class ThreadPool extends AbstractComponent { SizeValue capacity = settings.getAsSize("queue_size", defaultSettings.getAsSize("queue_size", new SizeValue(1000))); TimeValue waitTime = settings.getAsTime("wait_time", defaultSettings.getAsTime("wait_time", timeValueSeconds(60))); logger.debug("creating thread_pool [{}], type [{}], min [{}], size [{}], queue_size [{}], keep_alive [{}], wait_time [{}]", name, type, min, size, capacity.singles(), keepAlive, waitTime); - return DynamicExecutors.newBlockingThreadPool(min, size, keepAlive.millis(), (int) capacity.singles(), waitTime.millis(), threadFactory); + return EsExecutors.newBlockingExecutorService(min, size, keepAlive.millis(), TimeUnit.MILLISECONDS, threadFactory, (int) capacity.singles(), waitTime.millis(), TimeUnit.MILLISECONDS); } throw new ElasticSearchIllegalArgumentException("No type found [" + type + "], for [" + name + "]"); } diff --git a/src/test/java/org/elasticsearch/test/unit/common/util/concurrent/BlockingThreadPoolTest.java b/src/test/java/org/elasticsearch/test/unit/common/util/concurrent/BlockingThreadPoolTest.java deleted file mode 100644 index fa30161ee38..00000000000 --- a/src/test/java/org/elasticsearch/test/unit/common/util/concurrent/BlockingThreadPoolTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.unit.common.util.concurrent; - -import org.elasticsearch.common.util.concurrent.ThreadBarrier; -import org.elasticsearch.common.util.concurrent.TransferThreadPoolExecutor; -import org.testng.annotations.Test; - -import java.util.concurrent.Executors; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeUnit; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; - -/** - * - */ -@Test(enabled = false) -public class BlockingThreadPoolTest { - - @Test - public void testBlocking() throws Exception { - final int min = 2; - final int max = 4; - final long waitTime = 1000; //1 second - final ThreadBarrier barrier = new ThreadBarrier(max + 1); - - TransferThreadPoolExecutor pool = TransferThreadPoolExecutor.newBlockingExecutor(min, max, 60000, TimeUnit.MILLISECONDS, waitTime, TimeUnit.MILLISECONDS, 1, Executors.defaultThreadFactory()); - assertThat("Min property", pool.getCorePoolSize(), equalTo(min)); - assertThat("Max property", pool.getMaximumPoolSize(), equalTo(max)); - - for (int i = 0; i < max; ++i) { - pool.execute(new Runnable() { - public void run() { - try { - barrier.await(); - barrier.await(); - } catch (Throwable e) { - barrier.reset(e); - } - } - }); - - //wait until thread executes this task - //otherwise, a task might be queued - Thread.sleep(100); - } - - barrier.await(); - assertThat("wrong pool size", pool.getPoolSize(), equalTo(max)); - assertThat("wrong active size", pool.getActiveCount(), equalTo(max)); - - //Queue should be empty, lets occupy it's only free space - assertThat("queue isn't empty", pool.getQueueSize(), equalTo(0)); - pool.execute(new Runnable() { - public void run() { - //dummy task - } - }); - assertThat("queue isn't full", pool.getQueueSize(), equalTo(1)); - - //request should block since queue is full - try { - pool.execute(new Runnable() { - public void run() { - //dummy task - } - }); - assertThat("Should have thrown RejectedExecutionException", false, equalTo(true)); - } catch (RejectedExecutionException e) { - //caught expected exception - } - - barrier.await(); - pool.shutdown(); - } -} diff --git a/src/test/java/org/elasticsearch/test/unit/common/util/concurrent/ScalingThreadPoolTest.java b/src/test/java/org/elasticsearch/test/unit/common/util/concurrent/EsExecutorsTests.java similarity index 66% rename from src/test/java/org/elasticsearch/test/unit/common/util/concurrent/ScalingThreadPoolTest.java rename to src/test/java/org/elasticsearch/test/unit/common/util/concurrent/EsExecutorsTests.java index 89b6c318eff..3023595f449 100644 --- a/src/test/java/org/elasticsearch/test/unit/common/util/concurrent/ScalingThreadPoolTest.java +++ b/src/test/java/org/elasticsearch/test/unit/common/util/concurrent/EsExecutorsTests.java @@ -19,22 +19,22 @@ package org.elasticsearch.test.unit.common.util.concurrent; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadBarrier; -import org.elasticsearch.common.util.concurrent.TransferThreadPoolExecutor; import org.testng.annotations.Test; -import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThan; /** - * */ -@Test(enabled = false) -public class ScalingThreadPoolTest { +@Test +public class EsExecutorsTests { @Test public void testScaleUp() throws Exception { @@ -42,8 +42,7 @@ public class ScalingThreadPoolTest { final int max = 4; final ThreadBarrier barrier = new ThreadBarrier(max + 1); -// ThreadPoolExecutor pool = (ThreadPoolExecutor) DynamicExecutors.newScalingThreadPool(min, max, Long.MAX_VALUE); - TransferThreadPoolExecutor pool = TransferThreadPoolExecutor.newScalingExecutor(min, max, Long.MAX_VALUE, TimeUnit.NANOSECONDS, Executors.defaultThreadFactory()); + ThreadPoolExecutor pool = EsExecutors.newScalingExecutorService(min, max, 100, TimeUnit.DAYS, EsExecutors.daemonThreadFactory("test")); assertThat("Min property", pool.getCorePoolSize(), equalTo(min)); assertThat("Max property", pool.getMaximumPoolSize(), equalTo(max)); @@ -77,8 +76,7 @@ public class ScalingThreadPoolTest { final int max = 4; final ThreadBarrier barrier = new ThreadBarrier(max + 1); -// ThreadPoolExecutor pool = (ThreadPoolExecutor) DynamicExecutors.newScalingThreadPool(min, max, 0 /*keep alive*/); - TransferThreadPoolExecutor pool = TransferThreadPoolExecutor.newScalingExecutor(min, max, 0, TimeUnit.NANOSECONDS, Executors.defaultThreadFactory()); + ThreadPoolExecutor pool = EsExecutors.newScalingExecutorService(min, max, 10, TimeUnit.MILLISECONDS, EsExecutors.daemonThreadFactory("test")); assertThat("Min property", pool.getCorePoolSize(), equalTo(min)); assertThat("Max property", pool.getMaximumPoolSize(), equalTo(max)); @@ -108,34 +106,29 @@ public class ScalingThreadPoolTest { // assertThat("not all tasks completed", pool.getCompletedTaskCount(), equalTo((long) max)); assertThat("wrong active count", pool.getActiveCount(), equalTo(0)); //Assert.assertEquals("wrong pool size. ", min, pool.getPoolSize()); //BUG in ThreadPool - Bug ID: 6458662 - assertThat("idle threads didn't shrink below max. (" + pool.getPoolSize() + ")", pool.getPoolSize(), greaterThan(0)); + //assertThat("idle threads didn't stay above min (" + pool.getPoolSize() + ")", pool.getPoolSize(), greaterThan(0)); assertThat("idle threads didn't shrink below max. (" + pool.getPoolSize() + ")", pool.getPoolSize(), lessThan(max)); pool.shutdown(); } @Test - public void testScaleAbove() throws Exception { + public void testBlocking() throws Exception { final int min = 2; final int max = 4; - final int ntasks = 16; + final long waitTime = 1000; //1 second final ThreadBarrier barrier = new ThreadBarrier(max + 1); -// ThreadPoolExecutor pool = (ThreadPoolExecutor) DynamicExecutors.newScalingThreadPool(min, max, Long.MAX_VALUE); - TransferThreadPoolExecutor pool = TransferThreadPoolExecutor.newScalingExecutor(min, max, Long.MAX_VALUE, TimeUnit.NANOSECONDS, Executors.defaultThreadFactory()); + ThreadPoolExecutor pool = EsExecutors.newBlockingExecutorService(min, max, 60, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory("test"), 1, waitTime, TimeUnit.MILLISECONDS); assertThat("Min property", pool.getCorePoolSize(), equalTo(min)); assertThat("Max property", pool.getMaximumPoolSize(), equalTo(max)); - final AtomicInteger tasksExecuted = new AtomicInteger(); - for (int i = 0; i < ntasks; ++i) { - final int id = i; + for (int i = 0; i < max; ++i) { pool.execute(new Runnable() { public void run() { - tasksExecuted.incrementAndGet(); try { - if (id < max) { - barrier.await(); - } + barrier.await(); + barrier.await(); } catch (Throwable e) { barrier.reset(e); } @@ -147,14 +140,32 @@ public class ScalingThreadPoolTest { Thread.sleep(100); } - assertThat("wrong number of pooled tasks", pool.getQueueSize(), equalTo(ntasks - max)); barrier.await(); + assertThat("wrong pool size", pool.getPoolSize(), equalTo(max)); + assertThat("wrong active size", pool.getActiveCount(), equalTo(max)); - //wait around for one second - Thread.sleep(1000); - assertThat("tasks not complete", tasksExecuted.get(), equalTo(ntasks)); -// assertThat("didn't scale above core pool size. (" + pool.getLargestPoolSize() + ")", pool.getLargestPoolSize(), greaterThan(min)); -// assertThat("Largest pool size exceeds max. (" + pool.getLargestPoolSize() + ")", pool.getLargestPoolSize(), lessThanOrEqualTo(max)); + //Queue should be empty, lets occupy it's only free space + assertThat("queue isn't empty", pool.getQueue().size(), equalTo(0)); + pool.execute(new Runnable() { + public void run() { + //dummy task + } + }); + assertThat("queue isn't full", pool.getQueue().size(), equalTo(1)); + + //request should block since queue is full + try { + pool.execute(new Runnable() { + public void run() { + //dummy task + } + }); + assertThat("Should have thrown RejectedExecutionException", false, equalTo(true)); + } catch (RejectedExecutionException e) { + //caught expected exception + } + + barrier.await(); pool.shutdown(); } } From 2b838b808e43b46d9ba602f78bddb963f950a37f Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 20 Dec 2011 17:42:06 +0200 Subject: [PATCH 056/270] add another path trie test for wildcard vs. contant --- .../test/unit/common/path/PathTrieTests.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/test/java/org/elasticsearch/test/unit/common/path/PathTrieTests.java b/src/test/java/org/elasticsearch/test/unit/common/path/PathTrieTests.java index 249762c19b2..57a14f70f70 100644 --- a/src/test/java/org/elasticsearch/test/unit/common/path/PathTrieTests.java +++ b/src/test/java/org/elasticsearch/test/unit/common/path/PathTrieTests.java @@ -98,4 +98,17 @@ public class PathTrieTests { assertThat(trie.retrieve("/b/testX", params), equalTo("test2")); assertThat(params.get("name"), equalTo("testX")); } + + @Test + public void testPreferNonWildcardExecution() { + PathTrie trie = new PathTrie(); + trie.insert("{test}", "test1"); + trie.insert("b", "test2"); + trie.insert("{test}/a", "test3"); + trie.insert("b/a", "test4"); + + Map params = newHashMap(); + assertThat(trie.retrieve("/b", params), equalTo("test2")); + assertThat(trie.retrieve("/b/a", params), equalTo("test4")); + } } From a488424404e1bfbe9d353353f5c9d7da9959e222 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 21 Dec 2011 03:53:44 +0200 Subject: [PATCH 057/270] Analysis: Add phonetic encodder called `bm` or `beider_morse`, closes #1552. --- pom.xml | 2 +- .../phonetic/PhoneticTokenFilterFactory.java | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c2d40dd6936..694918e3ab3 100644 --- a/pom.xml +++ b/pom.xml @@ -124,7 +124,7 @@ commons-codec commons-codec - 1.5 + 1.6 compile diff --git a/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticTokenFilterFactory.java b/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticTokenFilterFactory.java index e8c4657dfe0..b90671f5519 100644 --- a/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticTokenFilterFactory.java +++ b/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticTokenFilterFactory.java @@ -21,6 +21,9 @@ package org.elasticsearch.index.analysis.phonetic; import org.apache.commons.codec.Encoder; import org.apache.commons.codec.language.*; +import org.apache.commons.codec.language.bm.BeiderMorseEncoder; +import org.apache.commons.codec.language.bm.NameType; +import org.apache.commons.codec.language.bm.RuleType; import org.apache.lucene.analysis.TokenStream; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.common.inject.Inject; @@ -67,6 +70,25 @@ public class PhoneticTokenFilterFactory extends AbstractTokenFilterFactory { DoubleMetaphone doubleMetaphone = new DoubleMetaphone(); doubleMetaphone.setMaxCodeLen(settings.getAsInt("max_code_len", doubleMetaphone.getMaxCodeLen())); this.encoder = doubleMetaphone; + } else if ("bm".equalsIgnoreCase(encoder) || "beider_morse".equalsIgnoreCase(encoder)) { + BeiderMorseEncoder bm = new BeiderMorseEncoder(); + String ruleType = settings.get("rule_type", "approx"); + if ("approx".equalsIgnoreCase(ruleType)) { + bm.setRuleType(RuleType.APPROX); + } else if ("exact".equalsIgnoreCase(ruleType)) { + bm.setRuleType(RuleType.EXACT); + } else { + throw new ElasticSearchIllegalArgumentException("No matching rule type [" + ruleType + "] for beider morse encoder"); + } + String nameType = settings.get("name_type", "generic"); + if ("GENERIC".equalsIgnoreCase(nameType)) { + bm.setNameType(NameType.GENERIC); + } else if ("ASHKENAZI".equalsIgnoreCase(nameType)) { + bm.setNameType(NameType.ASHKENAZI); + } else if ("SEPHARDIC".equalsIgnoreCase(nameType)) { + bm.setNameType(NameType.SEPHARDIC); + } + this.encoder = bm; } else { throw new ElasticSearchIllegalArgumentException("unknown encoder [" + encoder + "] for phonetic token filter"); } From ffca91dda02a23e7ff960e9b90a8dc7e9daec31d Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 21 Dec 2011 04:27:28 +0200 Subject: [PATCH 058/270] improve detection of which logging library to use, use slf4j if its the log4j jar file used (does not have setLevel method) --- .../org/elasticsearch/common/logging/ESLoggerFactory.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java b/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java index 869ef673827..345c497ff62 100644 --- a/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java +++ b/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java @@ -32,7 +32,9 @@ public abstract class ESLoggerFactory { static { try { - Class.forName("org.apache.log4j.Logger"); + Class loggerClazz = Class.forName("org.apache.log4j.Logger"); + // below will throw a NoSuchMethod failure with using slf4j log4j bridge + loggerClazz.getMethod("setLevel", Class.forName("org.apache.log4j.Level")); defaultFactory = new Log4jESLoggerFactory(); } catch (Throwable e) { // no log4j From 91b60f1d2f41747e9435d411faa068b1864615b0 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 21 Dec 2011 05:28:28 +0200 Subject: [PATCH 059/270] Generate pid file even when running in foreground mode, closes #1553. --- bin/elasticsearch | 10 +++---- .../elasticsearch/bootstrap/Bootstrap.java | 28 +++++++++++++++---- .../bootstrap/ElasticSearchF.java | 2 +- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/bin/elasticsearch b/bin/elasticsearch index 5c0d042df7b..c91587ee289 100755 --- a/bin/elasticsearch +++ b/bin/elasticsearch @@ -114,20 +114,20 @@ launch_service() es_parms="-Delasticsearch" if [ "x$pidpath" != "x" ]; then - es_parms="$es_parms -Des-pidfile=$pidpath" + es_parms="$es_parms -Des.pidfile=$pidpath" fi - # The es-daemon option will tell ElasticSearch to close stdout/stderr, - # but it's up to us not to background. + # The es-foreground option will tell ElasticSearch not to close stdout/stderr, but it's up to us not to background. if [ "x$foreground" != "x" ]; then - es_parms="$es_parms -Des-foreground=yes" + es_parms="$es_parms -Des.foreground=yes" exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS $es_parms -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" $props \ org.elasticsearch.bootstrap.ElasticSearch else # Startup ElasticSearch, background it, and write the pid. exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS $es_parms -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" $props \ org.elasticsearch.bootstrap.ElasticSearch <&- & - [ ! -z "$pidpath" ] && printf '%d' $! > "$pidpath" + # no longer need to write the pid, the elasticsearch process will do it + # [ ! -z "$pidpath" ] && printf '%d' $! > "$pidpath" fi return $? diff --git a/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index 4d7655de8eb..f15bafaefc0 100644 --- a/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -38,6 +38,7 @@ import org.elasticsearch.node.NodeBuilder; import org.elasticsearch.node.internal.InternalSettingsPerparer; import java.io.File; +import java.io.RandomAccessFile; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -142,9 +143,28 @@ public class Bootstrap { public static void main(String[] args) { System.setProperty("es.logger.prefix", ""); bootstrap = new Bootstrap(); - String pidFile = System.getProperty("es-pidfile"); + final String pidFile = System.getProperty("es.pidfile", System.getProperty("es-pidfile")); - boolean foreground = System.getProperty("es-foreground") != null; + if (pidFile != null) { + try { + File fPidFile = new File(pidFile); + if (fPidFile.getParentFile() != null) { + FileSystemUtils.mkdirs(fPidFile.getParentFile()); + } + RandomAccessFile rafPidFile = new RandomAccessFile(fPidFile, "rw"); + rafPidFile.writeBytes(Long.toString(JvmInfo.jvmInfo().pid()) + "\n"); + rafPidFile.close(); + + fPidFile.deleteOnExit(); + } catch (Exception e) { + String errorMessage = buildErrorMessage("pid", e); + System.err.println(errorMessage); + System.err.flush(); + System.exit(3); + } + } + + boolean foreground = System.getProperty("es.foreground", System.getProperty("es-foreground")) != null; // handle the wrapper system property, if its a service, don't run as a service if (System.getProperty("wrapper.service", "XXX").equalsIgnoreCase("true")) { foreground = false; @@ -180,10 +200,6 @@ public class Bootstrap { } bootstrap.setup(true, tuple); - if (pidFile != null) { - new File(pidFile).deleteOnExit(); - } - stage = "Startup"; bootstrap.start(); diff --git a/src/main/java/org/elasticsearch/bootstrap/ElasticSearchF.java b/src/main/java/org/elasticsearch/bootstrap/ElasticSearchF.java index 8a9a7f68985..5c9d33d32c2 100644 --- a/src/main/java/org/elasticsearch/bootstrap/ElasticSearchF.java +++ b/src/main/java/org/elasticsearch/bootstrap/ElasticSearchF.java @@ -11,7 +11,7 @@ public class ElasticSearchF { } public static void main(String[] args) { - System.setProperty("es-foreground", "yes"); + System.setProperty("es.foreground", "yes"); Bootstrap.main(args); } } From c918a68554edfbce80069c16bea870e5b5767b2a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 21 Dec 2011 07:04:49 +0200 Subject: [PATCH 060/270] better output --- src/main/java/org/elasticsearch/env/NodeEnvironment.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/src/main/java/org/elasticsearch/env/NodeEnvironment.java index eaee48520ec..c28043ed2df 100644 --- a/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -129,7 +129,7 @@ public class NodeEnvironment extends AbstractComponent { if (logger.isTraceEnabled()) { StringBuilder sb = new StringBuilder("node data locations details:\n"); for (File file : nodesFiles) { - sb.append(" -> ").append(file.getAbsolutePath()).append(", free_space [").append(new ByteSizeValue(file.getFreeSpace())).append(", usable_space [").append(new ByteSizeValue(file.getUsableSpace())).append("\n"); + sb.append(" -> ").append(file.getAbsolutePath()).append(", free_space [").append(new ByteSizeValue(file.getFreeSpace())).append("], usable_space [").append(new ByteSizeValue(file.getUsableSpace())).append("]\n"); } logger.trace(sb.toString()); } From 55d8d0d9c63fb8dcde923cf37b94d6485f47eec0 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 21 Dec 2011 23:24:55 +0200 Subject: [PATCH 061/270] Analyze API: Allow to execute it without pre-creating an index, and allow to build custom analyzer (tokenizer + token_filters), closes #1555. --- .../admin/indices/analyze/AnalyzeRequest.java | 67 ++++++++++++++-- .../analyze/TransportAnalyzeAction.java | 78 ++++++++++++++++++- .../TransportSingleCustomOperationAction.java | 34 +++++++- .../client/IndicesAdminClient.java | 11 ++- .../analyze/AnalyzeRequestBuilder.java | 40 ++++++++-- .../support/AbstractIndicesAdminClient.java | 8 +- .../indices/analyze/RestAnalyzeAction.java | 4 + .../indices/analyze/AnalyzeActionTests.java | 18 +++-- 8 files changed, 236 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java index 7b9ae0d1edf..78200018bc1 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java @@ -21,6 +21,7 @@ package org.elasticsearch.action.admin.indices.analyze; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.single.custom.SingleCustomOperationRequest; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -40,19 +41,32 @@ public class AnalyzeRequest extends SingleCustomOperationRequest { private String analyzer; + private String tokenizer; + + private String[] tokenFilters; + private String field; AnalyzeRequest() { } + /** + * Constructs a new analyzer request for the provided text. + * + * @param text The text to analyze + */ + public AnalyzeRequest(String text) { + this.text = text; + } + /** * Constructs a new analyzer request for the provided index and text. * * @param index The index name * @param text The text to analyze */ - public AnalyzeRequest(String index, String text) { + public AnalyzeRequest(@Nullable String index, String text) { this.index = index; this.text = text; } @@ -79,6 +93,24 @@ public class AnalyzeRequest extends SingleCustomOperationRequest { return this.analyzer; } + public AnalyzeRequest tokenizer(String tokenizer) { + this.tokenizer = tokenizer; + return this; + } + + public String tokenizer() { + return this.tokenizer; + } + + public AnalyzeRequest tokenFilters(String... tokenFilters) { + this.tokenFilters = tokenFilters; + return this; + } + + public String[] tokenFilters() { + return this.tokenFilters; + } + public AnalyzeRequest field(String field) { this.field = field; return this; @@ -101,9 +133,6 @@ public class AnalyzeRequest extends SingleCustomOperationRequest { @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = super.validate(); - if (index == null) { - validationException = addValidationError("index is missing", validationException); - } if (text == null) { validationException = addValidationError("text is missing", validationException); } @@ -113,11 +142,23 @@ public class AnalyzeRequest extends SingleCustomOperationRequest { @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - index = in.readUTF(); + if (in.readBoolean()) { + index = in.readUTF(); + } text = in.readUTF(); if (in.readBoolean()) { analyzer = in.readUTF(); } + if (in.readBoolean()) { + tokenizer = in.readUTF(); + } + int size = in.readVInt(); + if (size > 0) { + tokenFilters = new String[size]; + for (int i = 0; i < size; i++) { + tokenFilters[i] = in.readUTF(); + } + } if (in.readBoolean()) { field = in.readUTF(); } @@ -126,9 +167,23 @@ public class AnalyzeRequest extends SingleCustomOperationRequest { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeUTF(index); + if (index == null) { + out.writeBoolean(false); + out.writeUTF(index); + } else { + out.writeUTF(index); + } out.writeUTF(text); writeOption(out, analyzer); + writeOption(out, tokenizer); + if (tokenFilters == null) { + out.writeVInt(0); + } else { + out.writeVInt(tokenFilters.length); + for (String tokenFilter : tokenFilters) { + out.writeUTF(tokenFilter); + } + } writeOption(out, field); } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index db7d0b7f256..ecd4c68fdb3 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -35,10 +35,14 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.ShardsIterator; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.FastStringReader; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.analysis.*; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.service.IndexService; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.analysis.IndicesAnalysisService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -52,11 +56,14 @@ public class TransportAnalyzeAction extends TransportSingleCustomOperationAction private final IndicesService indicesService; + private final IndicesAnalysisService indicesAnalysisService; + @Inject public TransportAnalyzeAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, - IndicesService indicesService) { + IndicesService indicesService, IndicesAnalysisService indicesAnalysisService) { super(settings, threadPool, clusterService, transportService); this.indicesService = indicesService; + this.indicesAnalysisService = indicesAnalysisService; } @Override @@ -86,16 +93,27 @@ public class TransportAnalyzeAction extends TransportSingleCustomOperationAction @Override protected ShardsIterator shards(ClusterState clusterState, AnalyzeRequest request) { + if (request.index() == null) { + // just execute locally.... + return null; + } request.index(clusterState.metaData().concreteIndex(request.index())); return clusterState.routingTable().index(request.index()).randomAllActiveShardsIt(); } @Override protected AnalyzeResponse shardOperation(AnalyzeRequest request, int shardId) throws ElasticSearchException { - IndexService indexService = indicesService.indexServiceSafe(request.index()); + IndexService indexService = null; + if (request.index() != null) { + indexService = indicesService.indexServiceSafe(request.index()); + } Analyzer analyzer = null; + boolean closeAnalyzer = false; String field = null; if (request.field() != null) { + if (indexService == null) { + throw new ElasticSearchIllegalArgumentException("No index provided, and trying to analyzer based on a specific field which requires the index parameter"); + } FieldMapper fieldMapper = indexService.mapperService().smartNameFieldMapper(request.field()); if (fieldMapper != null) { analyzer = fieldMapper.indexAnalyzer(); @@ -106,9 +124,58 @@ public class TransportAnalyzeAction extends TransportSingleCustomOperationAction field = "_all"; } if (analyzer == null && request.analyzer() != null) { - analyzer = indexService.analysisService().analyzer(request.analyzer()); + if (indexService == null) { + analyzer = indicesAnalysisService.analyzer(request.analyzer()); + } else { + analyzer = indexService.analysisService().analyzer(request.analyzer()); + } + if (analyzer == null) { + throw new ElasticSearchIllegalArgumentException("failed to find analyzer [" + request.analyzer() + "]"); + } + } else if (request.tokenizer() != null) { + TokenizerFactory tokenizerFactory; + if (indexService == null) { + TokenizerFactoryFactory tokenizerFactoryFactory = indicesAnalysisService.tokenizerFactoryFactory(request.tokenizer()); + if (tokenizerFactoryFactory == null) { + throw new ElasticSearchIllegalArgumentException("failed to find global tokenizer under [" + request.tokenizer() + "]"); + } + tokenizerFactory = tokenizerFactoryFactory.create(request.tokenizer(), ImmutableSettings.Builder.EMPTY_SETTINGS); + } else { + tokenizerFactory = indexService.analysisService().tokenizer(request.tokenizer()); + if (tokenizerFactory == null) { + throw new ElasticSearchIllegalArgumentException("failed to find tokenizer under [" + request.tokenizer() + "]"); + } + } + TokenFilterFactory[] tokenFilterFactories = new TokenFilterFactory[0]; + if (request.tokenFilters() != null && request.tokenFilters().length > 0) { + tokenFilterFactories = new TokenFilterFactory[request.tokenFilters().length]; + for (int i = 0; i < request.tokenFilters().length; i++) { + String tokenFilterName = request.tokenFilters()[i]; + if (indexService == null) { + TokenFilterFactoryFactory tokenFilterFactoryFactory = indicesAnalysisService.tokenFilterFactoryFactory(tokenFilterName); + if (tokenFilterFactoryFactory == null) { + throw new ElasticSearchIllegalArgumentException("failed to find global token filter under [" + request.tokenizer() + "]"); + } + tokenFilterFactories[i] = tokenFilterFactoryFactory.create(tokenFilterName, ImmutableSettings.Builder.EMPTY_SETTINGS); + } else { + tokenFilterFactories[i] = indexService.analysisService().tokenFilter(tokenFilterName); + if (tokenFilterFactories[i] == null) { + throw new ElasticSearchIllegalArgumentException("failed to find token filter under [" + request.tokenizer() + "]"); + } + } + if (tokenFilterFactories[i] == null) { + throw new ElasticSearchIllegalArgumentException("failed to find token filter under [" + request.tokenizer() + "]"); + } + } + } + analyzer = new CustomAnalyzer(tokenizerFactory, new CharFilterFactory[0], tokenFilterFactories); + closeAnalyzer = true; } else if (analyzer == null) { - analyzer = indexService.analysisService().defaultIndexAnalyzer(); + if (indexService == null) { + analyzer = Lucene.STANDARD_ANALYZER; + } else { + analyzer = indexService.analysisService().defaultIndexAnalyzer(); + } } if (analyzer == null) { throw new ElasticSearchIllegalArgumentException("failed to find analyzer"); @@ -143,6 +210,9 @@ public class TransportAnalyzeAction extends TransportSingleCustomOperationAction // ignore } } + if (closeAnalyzer) { + analyzer.close(); + } } return new AnalyzeResponse(tokens); diff --git a/src/main/java/org/elasticsearch/action/support/single/custom/TransportSingleCustomOperationAction.java b/src/main/java/org/elasticsearch/action/support/single/custom/TransportSingleCustomOperationAction.java index 0fa2c652821..4d995c0416b 100644 --- a/src/main/java/org/elasticsearch/action/support/single/custom/TransportSingleCustomOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/single/custom/TransportSingleCustomOperationAction.java @@ -76,6 +76,9 @@ public abstract class TransportSingleCustomOperationActiontrue @@ -57,9 +92,4 @@ public class AnalyzeRequestBuilder extends BaseIndicesRequestBuilder listener) { client.analyze(request, listener); } - - public AnalyzeRequestBuilder setField(String field) { - request.field(field); - return this; - } } diff --git a/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java b/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java index c5b4f3c2329..a2ebcb7c352 100644 --- a/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java +++ b/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java @@ -40,6 +40,7 @@ import org.elasticsearch.client.action.admin.indices.status.IndicesStatusRequest import org.elasticsearch.client.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder; import org.elasticsearch.client.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; import org.elasticsearch.client.internal.InternalIndicesAdminClient; +import org.elasticsearch.common.Nullable; /** * @@ -132,10 +133,15 @@ public abstract class AbstractIndicesAdminClient implements InternalIndicesAdmin } @Override - public AnalyzeRequestBuilder prepareAnalyze(String index, String text) { + public AnalyzeRequestBuilder prepareAnalyze(@Nullable String index, String text) { return new AnalyzeRequestBuilder(this, index, text); } + @Override + public AnalyzeRequestBuilder prepareAnalyze(String text) { + return new AnalyzeRequestBuilder(this, null, text); + } + @Override public PutIndexTemplateRequestBuilder preparePutTemplate(String name) { return new PutIndexTemplateRequestBuilder(this, name); diff --git a/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java b/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java index 242cda862f9..49eb84808ef 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java @@ -44,7 +44,9 @@ public class RestAnalyzeAction extends BaseRestHandler { @Inject public RestAnalyzeAction(Settings settings, Client client, RestController controller) { super(settings, client); + controller.registerHandler(GET, "/_analyze", this); controller.registerHandler(GET, "/{index}/_analyze", this); + controller.registerHandler(POST, "/_analyze", this); controller.registerHandler(POST, "/{index}/_analyze", this); } @@ -67,6 +69,8 @@ public class RestAnalyzeAction extends BaseRestHandler { analyzeRequest.preferLocal(request.paramAsBoolean("prefer_local", analyzeRequest.preferLocalShard())); analyzeRequest.analyzer(request.param("analyzer")); analyzeRequest.field(request.param("field")); + analyzeRequest.tokenizer(request.param("tokenizer")); + analyzeRequest.tokenFilters(request.paramAsStringArray("token_filters", request.paramAsStringArray("filters", null))); client.admin().indices().analyze(analyzeRequest, new ActionListener() { @Override public void onResponse(AnalyzeResponse response) { diff --git a/src/test/java/org/elasticsearch/test/integration/indices/analyze/AnalyzeActionTests.java b/src/test/java/org/elasticsearch/test/integration/indices/analyze/AnalyzeActionTests.java index 9b540b326e2..949ac274f56 100644 --- a/src/test/java/org/elasticsearch/test/integration/indices/analyze/AnalyzeActionTests.java +++ b/src/test/java/org/elasticsearch/test/integration/indices/analyze/AnalyzeActionTests.java @@ -75,13 +75,21 @@ public class AnalyzeActionTests extends AbstractNodesTests { } } + @Test + public void analyzeWithNoIndex() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + + AnalyzeResponse analyzeResponse = client.admin().indices().prepareAnalyze("THIS IS A TEST").setAnalyzer("simple").execute().actionGet(); + assertThat(analyzeResponse.tokens().size(), equalTo(4)); + + analyzeResponse = client.admin().indices().prepareAnalyze("THIS IS A TEST").setTokenizer("keyword").setTokenFilters("lowercase").execute().actionGet(); + assertThat(analyzeResponse.tokens().size(), equalTo(1)); + assertThat(analyzeResponse.tokens().get(0).term(), equalTo("this is a test")); + } + @Test public void analyzerWithFieldOrTypeTests() throws Exception { - try { - client.admin().indices().prepareDelete("test").execute().actionGet(); - } catch (Exception e) { - // ignore - } + client.admin().indices().prepareDelete().execute().actionGet(); client.admin().indices().prepareCreate("test").execute().actionGet(); client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); From 52743a05fa8cd68dc17a78325b682d9f8234bb73 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 21 Dec 2011 23:36:16 +0200 Subject: [PATCH 062/270] rename setEncoder to setHighlighterEncoder, its not evident which encoder is refers to --- .../client/action/search/SearchRequestBuilder.java | 4 +--- .../search/highlight/HighlighterSearchTests.java | 6 ++++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java b/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java index a7d0a59d476..294e4f33b6f 100644 --- a/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java +++ b/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java @@ -42,8 +42,6 @@ import java.util.Map; /** * A search action request builder. - * - * */ public class SearchRequestBuilder extends BaseRequestBuilder { @@ -579,7 +577,7 @@ public class SearchRequestBuilder extends BaseRequestBuilder Date: Wed, 21 Dec 2011 23:48:16 +0200 Subject: [PATCH 063/270] Boolean Type: Consider `F` as false as well when searching on it, closes #1556. --- .../elasticsearch/index/mapper/core/BooleanFieldMapper.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java index e206420c86e..addff856fab 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java @@ -151,6 +151,9 @@ public class BooleanFieldMapper extends AbstractFieldMapper { if (value == null || value.length() == 0) { return "F"; } + if (value.length() == 1 && value.charAt(0) == 'F') { + return "F"; + } if (Booleans.parseBoolean(value, false)) { return "T"; } From fe4ba2ad559451ba1cdf61cb7832855e10a93b6e Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 22 Dec 2011 02:24:36 +0200 Subject: [PATCH 064/270] Improve multi field mapper with highlighting based on source, closes #1559. --- .../index/mapper/ContentPath.java | 13 ++ .../index/mapper/FieldMapper.java | 14 ++ .../mapper/core/AbstractFieldMapper.java | 2 +- .../mapper/multifield/MultiFieldMapper.java | 2 + .../search/highlight/HighlightPhase.java | 2 +- .../SourceScoreOrderFragmentsBuilder.java | 2 +- .../SourceSimpleFragmentsBuilder.java | 2 +- .../highlight/HighlighterSearchTests.java | 164 ++++++++++++++++++ 8 files changed, 197 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/mapper/ContentPath.java b/src/main/java/org/elasticsearch/index/mapper/ContentPath.java index 337b6bd16c7..3d923379ebc 100644 --- a/src/main/java/org/elasticsearch/index/mapper/ContentPath.java +++ b/src/main/java/org/elasticsearch/index/mapper/ContentPath.java @@ -41,6 +41,8 @@ public class ContentPath { private String[] path = new String[10]; + private String sourcePath; + public ContentPath() { this(0); } @@ -58,6 +60,7 @@ public class ContentPath { public void reset() { this.index = 0; + this.sourcePath = null; } public void add(String name) { @@ -96,4 +99,14 @@ public class ContentPath { public void pathType(Type type) { this.pathType = type; } + + public String sourcePath(String sourcePath) { + String orig = this.sourcePath; + this.sourcePath = sourcePath; + return orig; + } + + public String sourcePath() { + return this.sourcePath; + } } diff --git a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index b6802bbffb5..38a1f29007b 100644 --- a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -44,6 +44,8 @@ public interface FieldMapper { private final String fullName; + private final String sourcePath; + private final Term indexNameTermFactory; public Names(String name) { @@ -51,10 +53,15 @@ public interface FieldMapper { } public Names(String name, String indexName, String indexNameClean, String fullName) { + this(name, indexName, indexNameClean, fullName, fullName); + } + + public Names(String name, String indexName, String indexNameClean, String fullName, @Nullable String sourcePath) { this.name = name.intern(); this.indexName = indexName.intern(); this.indexNameClean = indexNameClean.intern(); this.fullName = fullName.intern(); + this.sourcePath = sourcePath == null ? this.fullName : sourcePath.intern(); this.indexNameTermFactory = new Term(indexName, ""); } @@ -87,6 +94,13 @@ public interface FieldMapper { return fullName; } + /** + * The dot path notation to extract the value from source. + */ + public String sourcePath() { + return sourcePath; + } + public Term createIndexNameTerm(String value) { return indexNameTermFactory.createTerm(value); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java index 0b8e7ff7f69..37bec35b5e8 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java @@ -182,7 +182,7 @@ public abstract class AbstractFieldMapper implements FieldMapper, Mapper { } protected Names buildNames(BuilderContext context) { - return new Names(name, buildIndexName(context), indexName == null ? name : indexName, buildFullName(context)); + return new Names(name, buildIndexName(context), indexName == null ? name : indexName, buildFullName(context), context.path().sourcePath()); } protected String buildIndexName(BuilderContext context) { diff --git a/src/main/java/org/elasticsearch/index/mapper/multifield/MultiFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/multifield/MultiFieldMapper.java index e7159aacde9..67bc4a1fc25 100644 --- a/src/main/java/org/elasticsearch/index/mapper/multifield/MultiFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/multifield/MultiFieldMapper.java @@ -82,6 +82,7 @@ public class MultiFieldMapper implements Mapper, AllFieldMapper.IncludeInAll { defaultMapper = defaultMapperBuilder.build(context); } + String origSourcePath = context.path().sourcePath(context.path().fullPathAsText(name)); context.path().add(name); Map mappers = new HashMap(); for (Mapper.Builder builder : mappersBuilders) { @@ -89,6 +90,7 @@ public class MultiFieldMapper implements Mapper, AllFieldMapper.IncludeInAll { mappers.put(mapper.name(), mapper); } context.path().remove(); + context.path().sourcePath(origSourcePath); context.path().pathType(origPathType); diff --git a/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java b/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java index fbf4fd21323..e07881c0c3f 100644 --- a/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java +++ b/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java @@ -182,7 +182,7 @@ public class HighlightPhase implements FetchSubPhase { SearchLookup lookup = context.lookup(); lookup.setNextReader(hitContext.reader()); lookup.setNextDocId(hitContext.docId()); - textsToHighlight = lookup.source().extractRawValues(mapper.names().fullName()); + textsToHighlight = lookup.source().extractRawValues(mapper.names().sourcePath()); } // a HACK to make highlighter do highlighting, even though its using the single frag list builder diff --git a/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java b/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java index dd5639fdb7a..a13e95704c5 100644 --- a/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java +++ b/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java @@ -52,7 +52,7 @@ public class SourceScoreOrderFragmentsBuilder extends ScoreOrderFragmentsBuilder lookup.setNextReader(reader); lookup.setNextDocId(docId); - List values = lookup.source().extractRawValues(mapper.names().fullName()); + List values = lookup.source().extractRawValues(mapper.names().sourcePath()); Field[] fields = new Field[values.size()]; for (int i = 0; i < values.size(); i++) { fields[i] = new Field(mapper.names().indexName(), values.get(i).toString(), Field.Store.NO, Field.Index.ANALYZED); diff --git a/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java b/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java index 26894ed0c1e..48bcc18f715 100644 --- a/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java +++ b/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java @@ -54,7 +54,7 @@ public class SourceSimpleFragmentsBuilder extends SimpleFragmentsBuilder { lookup.setNextReader(reader); lookup.setNextDocId(docId); - List values = lookup.source().extractRawValues(mapper.names().fullName()); + List values = lookup.source().extractRawValues(mapper.names().sourcePath()); if (values.isEmpty()) { return EMPTY_FIELDS; } diff --git a/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java index f307c77942c..6a3d5f9bab6 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java @@ -527,4 +527,168 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("highlighting test for *&? elasticsearch ")); } } + + @Test + public void testMultiMapperVectorWithStore() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + + client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 2)) + .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") + .startObject("title").field("type", "multi_field").startObject("fields") + .startObject("title").field("type", "string").field("store", "yes").field("term_vector", "with_positions_offsets").endObject() + .startObject("key").field("type", "string").field("store", "yes").field("term_vector", "with_positions_offsets").field("analyzer", "whitespace").endObject() + .endObject().endObject() + .endObject().endObject().endObject()) + .execute().actionGet(); + + client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); + + client.prepareIndex("test", "type1", "1").setSource("title", "this is a test").execute().actionGet(); + client.admin().indices().prepareRefresh().execute().actionGet(); + + // simple search on body with standard analyzer with a simple field query + SearchResponse search = client.prepareSearch() + .setQuery(fieldQuery("title", "this is a test")) + .setHighlighterEncoder("html") + .addHighlightedField("title", 50, 1) + .execute().actionGet(); + assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0)); + + SearchHit hit = search.hits().getAt(0); + assertThat(hit.highlightFields().get("title").fragments()[0], equalTo(" is a test ")); + + // search on title.key and highlight on title + search = client.prepareSearch() + .setQuery(fieldQuery("title.key", "this is a test")) + .setHighlighterEncoder("html") + .addHighlightedField("title.key", 50, 1) + .execute().actionGet(); + assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0)); + + hit = search.hits().getAt(0); + assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("this is a test ")); + } + + @Test + public void testMultiMapperVectorFromSource() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + + client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 2)) + .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") + .startObject("title").field("type", "multi_field").startObject("fields") + .startObject("title").field("type", "string").field("store", "no").field("term_vector", "with_positions_offsets").endObject() + .startObject("key").field("type", "string").field("store", "no").field("term_vector", "with_positions_offsets").field("analyzer", "whitespace").endObject() + .endObject().endObject() + .endObject().endObject().endObject()) + .execute().actionGet(); + + client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); + + client.prepareIndex("test", "type1", "1").setSource("title", "this is a test").execute().actionGet(); + client.admin().indices().prepareRefresh().execute().actionGet(); + + // simple search on body with standard analyzer with a simple field query + SearchResponse search = client.prepareSearch() + .setQuery(fieldQuery("title", "this is a test")) + .setHighlighterEncoder("html") + .addHighlightedField("title", 50, 1) + .execute().actionGet(); + assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0)); + + SearchHit hit = search.hits().getAt(0); + assertThat(hit.highlightFields().get("title").fragments()[0], equalTo(" is a test ")); + + // search on title.key and highlight on title.key + search = client.prepareSearch() + .setQuery(fieldQuery("title.key", "this is a test")) + .setHighlighterEncoder("html") + .addHighlightedField("title.key", 50, 1) + .execute().actionGet(); + assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0)); + + hit = search.hits().getAt(0); + assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("this is a test ")); + } + + @Test + public void testMultiMapperNoVectorWithStore() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + + client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 2)) + .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") + .startObject("title").field("type", "multi_field").startObject("fields") + .startObject("title").field("type", "string").field("store", "yes").field("term_vector", "no").endObject() + .startObject("key").field("type", "string").field("store", "yes").field("term_vector", "no").field("analyzer", "whitespace").endObject() + .endObject().endObject() + .endObject().endObject().endObject()) + .execute().actionGet(); + + client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); + + client.prepareIndex("test", "type1", "1").setSource("title", "this is a test").execute().actionGet(); + client.admin().indices().prepareRefresh().execute().actionGet(); + + // simple search on body with standard analyzer with a simple field query + SearchResponse search = client.prepareSearch() + .setQuery(fieldQuery("title", "this is a test")) + .setHighlighterEncoder("html") + .addHighlightedField("title", 50, 1) + .execute().actionGet(); + assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0)); + + SearchHit hit = search.hits().getAt(0); + assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("this is a test")); + + // search on title.key and highlight on title + search = client.prepareSearch() + .setQuery(fieldQuery("title.key", "this is a test")) + .setHighlighterEncoder("html") + .addHighlightedField("title.key", 50, 1) + .execute().actionGet(); + assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0)); + + hit = search.hits().getAt(0); + assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("this is a test")); + } + + @Test + public void testMultiMapperNoVectorFromSource() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + + client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 2)) + .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") + .startObject("title").field("type", "multi_field").startObject("fields") + .startObject("title").field("type", "string").field("store", "no").field("term_vector", "no").endObject() + .startObject("key").field("type", "string").field("store", "no").field("term_vector", "no").field("analyzer", "whitespace").endObject() + .endObject().endObject() + .endObject().endObject().endObject()) + .execute().actionGet(); + + client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); + + client.prepareIndex("test", "type1", "1").setSource("title", "this is a test").execute().actionGet(); + client.admin().indices().prepareRefresh().execute().actionGet(); + + // simple search on body with standard analyzer with a simple field query + SearchResponse search = client.prepareSearch() + .setQuery(fieldQuery("title", "this is a test")) + .setHighlighterEncoder("html") + .addHighlightedField("title", 50, 1) + .execute().actionGet(); + assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0)); + + SearchHit hit = search.hits().getAt(0); + assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("this is a test")); + + // search on title.key and highlight on title.key + search = client.prepareSearch() + .setQuery(fieldQuery("title.key", "this is a test")) + .setHighlighterEncoder("html") + .addHighlightedField("title.key", 50, 1) + .execute().actionGet(); + assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0)); + + hit = search.hits().getAt(0); + assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("this is a test")); + } } From 61b256262308d3483d11e0f2f8a738a6e918f534 Mon Sep 17 00:00:00 2001 From: "jayson.minard" Date: Thu, 22 Dec 2011 00:19:38 -0800 Subject: [PATCH 065/270] Query DSL: custom_filters_score allow score_mode for "min" and "multiply", closes #1560 --- .../function/FiltersFunctionScoreQuery.java | 30 +++++++++++++++++-- .../query/CustomFiltersScoreQueryParser.java | 4 +++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java b/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java index a54a2711d9b..2c682f556c0 100644 --- a/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java +++ b/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java @@ -69,7 +69,7 @@ public class FiltersFunctionScoreQuery extends Query { } } - public static enum ScoreMode {First, Avg, Max, Total} + public static enum ScoreMode {First, Avg, Max, Total, Min, Multiply} Query subQuery; final FilterFunction[] filterFunctions; @@ -178,7 +178,9 @@ public class FiltersFunctionScoreQuery extends Query { } else { int count = 0; float total = 0; + float multiply = 1; float max = Float.NEGATIVE_INFINITY; + float min = Float.POSITIVE_INFINITY; ArrayList filtersExplanations = new ArrayList(); for (FilterFunction filterFunction : filterFunctions) { DocSet docSet = DocSets.convert(reader, filterFunction.filter.getDocIdSet(reader)); @@ -188,7 +190,9 @@ public class FiltersFunctionScoreQuery extends Query { float sc = functionExplanation.getValue(); count++; total += sc; + multiply *= sc; max = Math.max(sc, max); + min = Math.min(sc, min); Explanation res = new ComplexExplanation(true, sc, "custom score, product of:"); res.addDetail(new Explanation(1.0f, "match filter: " + filterFunction.filter.toString())); res.addDetail(functionExplanation); @@ -205,9 +209,15 @@ public class FiltersFunctionScoreQuery extends Query { case Max: sc = max; break; + case Min: + sc = min; + break; case Total: sc = total; break; + case Multiply: + sc = multiply; + break; } sc *= getValue(); Explanation res = new ComplexExplanation(true, sc, "custom score, score mode [" + scoreMode.toString().toLowerCase() + "]"); @@ -279,12 +289,25 @@ public class FiltersFunctionScoreQuery extends Query { if (maxScore != Float.NEGATIVE_INFINITY) { score = maxScore; } + } else if (scoreMode == ScoreMode.Min) { + float minScore = Float.POSITIVE_INFINITY; + for (int i = 0; i < filterFunctions.length; i++) { + if (docSets[i].get(docId)) { + minScore = Math.min(filterFunctions[i].function.score(docId, score), minScore); + } + } + if (minScore != Float.POSITIVE_INFINITY) { + score = minScore; + } } else { // Avg / Total float totalScore = 0.0f; + float multiplicativeScore = 1.0f; int count = 0; for (int i = 0; i < filterFunctions.length; i++) { if (docSets[i].get(docId)) { - totalScore += filterFunctions[i].function.score(docId, score); + float tempScore = filterFunctions[i].function.score(docId, score); + totalScore += tempScore; + multiplicativeScore *= tempScore; count++; } } @@ -293,6 +316,9 @@ public class FiltersFunctionScoreQuery extends Query { if (scoreMode == ScoreMode.Avg) { score /= count; } + else if (scoreMode == ScoreMode.Multiply) { + score = multiplicativeScore; + } } } return subQueryWeight * score; diff --git a/src/main/java/org/elasticsearch/index/query/CustomFiltersScoreQueryParser.java b/src/main/java/org/elasticsearch/index/query/CustomFiltersScoreQueryParser.java index c2a95572008..2d0d0249a35 100644 --- a/src/main/java/org/elasticsearch/index/query/CustomFiltersScoreQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/CustomFiltersScoreQueryParser.java @@ -120,8 +120,12 @@ public class CustomFiltersScoreQueryParser implements QueryParser { scoreMode = FiltersFunctionScoreQuery.ScoreMode.Avg; } else if ("max".equals(sScoreMode)) { scoreMode = FiltersFunctionScoreQuery.ScoreMode.Max; + } else if ("min".equals(sScoreMode)) { + scoreMode = FiltersFunctionScoreQuery.ScoreMode.Min; } else if ("total".equals(sScoreMode)) { scoreMode = FiltersFunctionScoreQuery.ScoreMode.Total; + } else if ("multiply".equals(sScoreMode)) { + scoreMode = FiltersFunctionScoreQuery.ScoreMode.Multiply; } else if ("first".equals(sScoreMode)) { scoreMode = FiltersFunctionScoreQuery.ScoreMode.First; } else { From 415ee6425a5bb651b803ef5c7d2118ad5f76733d Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 22 Dec 2011 14:25:54 +0200 Subject: [PATCH 066/270] Allow search to continue when sort field is missing from type mapping, closes #1558. --- .../search/sort/FieldSortBuilder.java | 16 ++++++++-- .../search/sort/SortParseElement.java | 14 +++++--- .../search/sort/SimpleSortTests.java | 32 +++++++++++++++++++ 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index de0a8faccae..a071bb60098 100644 --- a/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -25,8 +25,6 @@ import java.io.IOException; /** * A sort builder to sort based on a document field. - * - * */ public class FieldSortBuilder extends SortBuilder { @@ -36,6 +34,8 @@ public class FieldSortBuilder extends SortBuilder { private Object missing; + private Boolean ignoreUnampped; + /** * Constructs a new sort based on a document field. * @@ -64,6 +64,15 @@ public class FieldSortBuilder extends SortBuilder { return this; } + /** + * Sets if the field does not exists in the index, it should be ignored and not sorted by or not. Defaults + * to false (not ignoring). + */ + public FieldSortBuilder ignoreUnmapped(boolean ignoreUnmapped) { + this.ignoreUnampped = ignoreUnmapped; + return this; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(fieldName); @@ -73,6 +82,9 @@ public class FieldSortBuilder extends SortBuilder { if (missing != null) { builder.field("missing", missing); } + if (ignoreUnampped != null) { + builder.field("ignore_unmapped", ignoreUnampped); + } builder.endObject(); return builder; } diff --git a/src/main/java/org/elasticsearch/search/sort/SortParseElement.java b/src/main/java/org/elasticsearch/search/sort/SortParseElement.java index ec809692ae9..bbf2a51b9df 100644 --- a/src/main/java/org/elasticsearch/search/sort/SortParseElement.java +++ b/src/main/java/org/elasticsearch/search/sort/SortParseElement.java @@ -69,7 +69,7 @@ public class SortParseElement implements SearchParseElement { if (token == XContentParser.Token.START_OBJECT) { addCompoundSortField(parser, context, sortFields); } else if (token == XContentParser.Token.VALUE_STRING) { - addSortField(context, sortFields, parser.text(), false, null); + addSortField(context, sortFields, parser.text(), false, false, null); } } } else { @@ -88,6 +88,7 @@ public class SortParseElement implements SearchParseElement { boolean reverse = false; String missing = null; String innerJsonName = null; + boolean ignoreUnmapped = false; token = parser.nextToken(); if (token == XContentParser.Token.VALUE_STRING) { String direction = parser.text(); @@ -96,7 +97,7 @@ public class SortParseElement implements SearchParseElement { } else if (direction.equals("desc")) { reverse = !SCORE_FIELD_NAME.equals(fieldName); } - addSortField(context, sortFields, fieldName, reverse, missing); + addSortField(context, sortFields, fieldName, reverse, ignoreUnmapped, missing); } else { if (parsers.containsKey(fieldName)) { sortFields.add(parsers.get(fieldName).parse(parser, context)); @@ -115,17 +116,19 @@ public class SortParseElement implements SearchParseElement { } } else if ("missing".equals(innerJsonName)) { missing = parser.textOrNull(); + } else if ("ignore_unmapped".equals(innerJsonName) || "ignoreUnmapped".equals(innerJsonName)) { + ignoreUnmapped = parser.booleanValue(); } } } - addSortField(context, sortFields, fieldName, reverse, missing); + addSortField(context, sortFields, fieldName, reverse, ignoreUnmapped, missing); } } } } } - private void addSortField(SearchContext context, List sortFields, String fieldName, boolean reverse, @Nullable final String missing) { + private void addSortField(SearchContext context, List sortFields, String fieldName, boolean reverse, boolean ignoreUnmapped, @Nullable final String missing) { if (SCORE_FIELD_NAME.equals(fieldName)) { if (reverse) { sortFields.add(SORT_SCORE_REVERSE); @@ -141,6 +144,9 @@ public class SortParseElement implements SearchParseElement { } else { FieldMapper fieldMapper = context.smartNameFieldMapper(fieldName); if (fieldMapper == null) { + if (ignoreUnmapped) { + return; + } throw new SearchParseException(context, "No mapping found for [" + fieldName + "] in order to sort on"); } sortFields.add(new SortField(fieldMapper.names().indexName(), fieldMapper.fieldDataType().newFieldComparatorSource(context.fieldDataCache(), missing), reverse)); diff --git a/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java b/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java index 2f99775cbc6..8fa893252d8 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.test.integration.search.sort; +import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.client.Client; @@ -563,4 +564,35 @@ public class SimpleSortTests extends AbstractNodesTests { assertThat(searchResponse.hits().getAt(1).id(), equalTo("1")); assertThat(searchResponse.hits().getAt(2).id(), equalTo("3")); } + + @Test + public void testIgnoreUnmapped() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + + client.prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() + .field("id", "1") + .field("i_value", -1) + .field("d_value", -1.1) + .endObject()).execute().actionGet(); + + client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); + + logger.info("--> sort with an unmapped field, verify it fails"); + try { + SearchResponse searchResponse = client.prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("kkk")) + .execute().actionGet(); + assert false; + } catch (SearchPhaseExecutionException e) { + + } + + SearchResponse searchResponse = client.prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("kkk").ignoreUnmapped(true)) + .execute().actionGet(); + + assertThat(searchResponse.failedShards(), equalTo(0)); + } } From a0fb6f3d92b451348ac1d73c625a9db885cfc4b3 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 22 Dec 2011 20:31:32 +0200 Subject: [PATCH 067/270] sync with latest patch of byte buffer directory submitted ot Lucene, seems to pass all Luceen tests --- .../store/bytebuffer/ByteBufferAllocator.java | 36 +++--- .../store/bytebuffer/ByteBufferDirectory.java | 48 ++++---- .../store/bytebuffer/ByteBufferFile.java | 53 +++++---- .../bytebuffer/ByteBufferIndexInput.java | 111 +++++++++++------- .../bytebuffer/ByteBufferIndexOutput.java | 42 ++++--- .../CachingByteBufferAllocator.java | 36 +++--- .../bytebuffer/PlainByteBufferAllocator.java | 36 +++--- 7 files changed, 196 insertions(+), 166 deletions(-) diff --git a/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferAllocator.java b/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferAllocator.java index 6c78edaf65d..27b4a1a2b71 100644 --- a/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferAllocator.java +++ b/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferAllocator.java @@ -1,24 +1,22 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - package org.apache.lucene.store.bytebuffer; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import java.io.IOException; import java.lang.reflect.Method; import java.nio.ByteBuffer; diff --git a/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferDirectory.java b/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferDirectory.java index 362e8e8fd59..657f2e1c363 100644 --- a/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferDirectory.java +++ b/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferDirectory.java @@ -1,24 +1,22 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - package org.apache.lucene.store.bytebuffer; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; @@ -41,8 +39,6 @@ import java.util.concurrent.ConcurrentHashMap; *

*

If constructed with {@link ByteBufferAllocator}, it allows to control the allocation and release of * byte buffer. For example, custom implementations can include caching of byte buffers. - * - * */ public class ByteBufferDirectory extends Directory { @@ -110,7 +106,7 @@ public class ByteBufferDirectory extends Directory { do { try { Thread.sleep(0, 1); - } catch (InterruptedException ie) { + } catch (java.lang.InterruptedException ie) { // In 3.0 we will change this to throw // InterruptedException instead Thread.currentThread().interrupt(); @@ -127,7 +123,7 @@ public class ByteBufferDirectory extends Directory { ByteBufferFile file = files.remove(name); if (file == null) throw new FileNotFoundException(name); - file.clean(); + file.delete(); } @Override @@ -147,9 +143,9 @@ public class ByteBufferDirectory extends Directory { ByteBufferFile file = new ByteBufferFile(this, allocator.sizeInBytes(allocatorType)); ByteBufferFile existing = files.put(name, file); if (existing != null) { - existing.clean(); + existing.delete(); } - return new ByteBufferIndexOutput(allocator, allocatorType, file); + return new ByteBufferIndexOutput(name, allocator, allocatorType, file); } @Override @@ -157,7 +153,7 @@ public class ByteBufferDirectory extends Directory { ByteBufferFile file = files.get(name); if (file == null) throw new FileNotFoundException(name); - return new ByteBufferIndexInput(file); + return new ByteBufferIndexInput(name, file); } @Override diff --git a/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferFile.java b/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferFile.java index 215a403617b..92627caa2c7 100644 --- a/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferFile.java +++ b/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferFile.java @@ -1,29 +1,27 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - package org.apache.lucene.store.bytebuffer; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import java.nio.ByteBuffer; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; /** - * */ public class ByteBufferFile { @@ -35,6 +33,8 @@ public class ByteBufferFile { // This is publicly modifiable via Directory.touchFile(), so direct access not supported private volatile long lastModified = System.currentTimeMillis(); + private final AtomicInteger refCount = new AtomicInteger(1); + public ByteBufferFile(ByteBufferDirectory dir, int bufferSize) { this.dir = dir; this.bufferSize = bufferSize; @@ -70,8 +70,17 @@ public class ByteBufferFile { return buffers.size(); } - void clean() { - if (buffers != null) { + void delete() { + decRef(); + } + + void incRef() { + refCount.incrementAndGet(); + } + + void decRef() { + if (refCount.decrementAndGet() == 0) { + length = 0; for (ByteBuffer buffer : buffers) { dir.releaseBuffer(buffer); } diff --git a/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferIndexInput.java b/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferIndexInput.java index 520650d54e0..7cb186dee69 100644 --- a/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferIndexInput.java +++ b/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferIndexInput.java @@ -19,11 +19,12 @@ package org.apache.lucene.store.bytebuffer; import org.apache.lucene.store.IndexInput; +import java.io.EOFException; import java.io.IOException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; /** - * */ public class ByteBufferIndexInput extends IndexInput { @@ -38,8 +39,12 @@ public class ByteBufferIndexInput extends IndexInput { private long bufferStart; private final int BUFFER_SIZE; - public ByteBufferIndexInput(ByteBufferFile file) throws IOException { + private volatile boolean closed = false; + + public ByteBufferIndexInput(String name, ByteBufferFile file) throws IOException { + super("BBIndexInput(name=" + name + ")"); this.file = file; + this.file.incRef(); this.length = file.getLength(); this.BUFFER_SIZE = file.bufferSize; @@ -51,7 +56,13 @@ public class ByteBufferIndexInput extends IndexInput { @Override public void close() { - // nothing to do here + // we protected from double closing the index input since + // some tests do that... + if (closed) { + return; + } + closed = true; + file.decRef(); } @Override @@ -59,32 +70,38 @@ public class ByteBufferIndexInput extends IndexInput { return length; } -// @Override -// public short readShort() throws IOException { -// try { -// return currentBuffer.getShort(); -// } catch (BufferUnderflowException e) { -// return super.readShort(); -// } -// } -// -// @Override -// public int readInt() throws IOException { -// try { -// return currentBuffer.getInt(); -// } catch (BufferUnderflowException e) { -// return super.readInt(); -// } -// } -// -// @Override -// public long readLong() throws IOException { -// try { -// return currentBuffer.getLong(); -// } catch (BufferUnderflowException e) { -// return super.readLong(); -// } -// } + @Override + public short readShort() throws IOException { + try { + currentBuffer.mark(); + return currentBuffer.getShort(); + } catch (BufferUnderflowException e) { + currentBuffer.reset(); + return super.readShort(); + } + } + + @Override + public int readInt() throws IOException { + try { + currentBuffer.mark(); + return currentBuffer.getInt(); + } catch (BufferUnderflowException e) { + currentBuffer.reset(); + return super.readInt(); + } + } + + @Override + public long readLong() throws IOException { + try { + currentBuffer.mark(); + return currentBuffer.getLong(); + } catch (BufferUnderflowException e) { + currentBuffer.reset(); + return super.readLong(); + } + } @Override public byte readByte() throws IOException { @@ -120,27 +137,28 @@ public class ByteBufferIndexInput extends IndexInput { public void seek(long pos) throws IOException { if (currentBuffer == EMPTY_BUFFER || pos < bufferStart || pos >= bufferStart + BUFFER_SIZE) { currentBufferIndex = (int) (pos / BUFFER_SIZE); - if (currentBufferIndex >= file.numBuffers()) { - // if we are past EOF, don't throw one here, instead, move it to the last position in the last buffer - currentBufferIndex = file.numBuffers() - 1; - currentBuffer = currentBufferIndex == -1 ? EMPTY_BUFFER : file.getBuffer(currentBufferIndex); - currentBuffer.position(currentBuffer.limit()); - return; - } else { - switchCurrentBuffer(false); - } + switchCurrentBuffer(false); } try { currentBuffer.position((int) (pos % BUFFER_SIZE)); + // Grrr, need to wrap in IllegalArgumentException since tests (if not other places) + // expect an IOException... } catch (IllegalArgumentException e) { - currentBuffer.position(currentBuffer.limit()); + IOException ioException = new IOException("seeking past position"); + ioException.initCause(e); + throw ioException; } } private void switchCurrentBuffer(boolean enforceEOF) throws IOException { if (currentBufferIndex >= file.numBuffers()) { + // end of file reached, no more buffers left if (enforceEOF) { - throw new IOException("Read past EOF"); + throw new EOFException("Read past EOF (resource: " + this + ")"); + } else { + // Force EOF if a read takes place at this position + currentBufferIndex--; + currentBuffer.position(currentBuffer.limit()); } } else { ByteBuffer buffer = file.getBuffer(currentBufferIndex); @@ -152,8 +170,16 @@ public class ByteBufferIndexInput extends IndexInput { long buflen = length - bufferStart; if (buflen < BUFFER_SIZE) { currentBuffer.limit((int) buflen); - if (enforceEOF && buflen == 0) { - throw new IOException("Read past EOF"); + } + + // we need to enforce EOF here as well... + if (!currentBuffer.hasRemaining()) { + if (enforceEOF) { + throw new EOFException("Read past EOF (resource: " + this + ")"); + } else { + // Force EOF if a read takes place at this position + currentBufferIndex--; + currentBuffer.position(currentBuffer.limit()); } } } @@ -162,6 +188,7 @@ public class ByteBufferIndexInput extends IndexInput { @Override public Object clone() { ByteBufferIndexInput cloned = (ByteBufferIndexInput) super.clone(); + cloned.file.incRef(); // inc ref on cloned one if (currentBuffer != EMPTY_BUFFER) { cloned.currentBuffer = currentBuffer.asReadOnlyBuffer(); cloned.currentBuffer.position(currentBuffer.position()); diff --git a/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferIndexOutput.java b/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferIndexOutput.java index ea886510c69..7d0f66eeb29 100644 --- a/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferIndexOutput.java +++ b/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferIndexOutput.java @@ -1,23 +1,22 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +package org.apache.lucene.store.bytebuffer; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -package org.apache.lucene.store.bytebuffer; import org.apache.lucene.store.IndexOutput; @@ -28,6 +27,9 @@ import java.nio.ByteBuffer; */ public class ByteBufferIndexOutput extends IndexOutput { + private final static ByteBuffer EMPTY_BUFFER = ByteBuffer.allocate(0).asReadOnlyBuffer(); + + private final String name; private final ByteBufferAllocator allocator; private final ByteBufferAllocator.Type allocatorType; private final int BUFFER_SIZE; @@ -38,13 +40,15 @@ public class ByteBufferIndexOutput extends IndexOutput { private long bufferStart; - public ByteBufferIndexOutput(ByteBufferAllocator allocator, ByteBufferAllocator.Type allocatorType, ByteBufferFile file) throws IOException { + public ByteBufferIndexOutput(String name, ByteBufferAllocator allocator, ByteBufferAllocator.Type allocatorType, ByteBufferFile file) throws IOException { + this.name = name; this.allocator = allocator; this.allocatorType = allocatorType; this.BUFFER_SIZE = file.bufferSize; this.file = file; - // create the first buffer we write to - switchCurrentBuffer(); + + currentBufferIndex = -1; + currentBuffer = EMPTY_BUFFER; } @Override diff --git a/src/main/java/org/apache/lucene/store/bytebuffer/CachingByteBufferAllocator.java b/src/main/java/org/apache/lucene/store/bytebuffer/CachingByteBufferAllocator.java index bb6d96d82fa..574ef20c576 100644 --- a/src/main/java/org/apache/lucene/store/bytebuffer/CachingByteBufferAllocator.java +++ b/src/main/java/org/apache/lucene/store/bytebuffer/CachingByteBufferAllocator.java @@ -1,24 +1,22 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - package org.apache.lucene.store.bytebuffer; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import java.io.IOException; import java.nio.ByteBuffer; import java.util.concurrent.ArrayBlockingQueue; diff --git a/src/main/java/org/apache/lucene/store/bytebuffer/PlainByteBufferAllocator.java b/src/main/java/org/apache/lucene/store/bytebuffer/PlainByteBufferAllocator.java index 8d256024db2..e800b618d2d 100644 --- a/src/main/java/org/apache/lucene/store/bytebuffer/PlainByteBufferAllocator.java +++ b/src/main/java/org/apache/lucene/store/bytebuffer/PlainByteBufferAllocator.java @@ -1,24 +1,22 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - package org.apache.lucene.store.bytebuffer; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import java.io.IOException; import java.nio.ByteBuffer; From 3d9e87201e52a2595c14a8dec3313c9595151a50 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 22 Dec 2011 20:42:37 +0200 Subject: [PATCH 068/270] Cluster metadata files destroyed when using blob store gateway causing data loss, closes #1564, --- .../gateway/shared/SharedStorageGateway.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/gateway/shared/SharedStorageGateway.java b/src/main/java/org/elasticsearch/gateway/shared/SharedStorageGateway.java index 8d3ce981cde..e2f121d946e 100644 --- a/src/main/java/org/elasticsearch/gateway/shared/SharedStorageGateway.java +++ b/src/main/java/org/elasticsearch/gateway/shared/SharedStorageGateway.java @@ -32,6 +32,12 @@ import org.elasticsearch.gateway.Gateway; import org.elasticsearch.gateway.GatewayException; import org.elasticsearch.threadpool.ThreadPool; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +import static java.util.concurrent.Executors.newSingleThreadExecutor; +import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory; + /** * */ @@ -41,6 +47,8 @@ public abstract class SharedStorageGateway extends AbstractLifecycleComponent Date: Thu, 22 Dec 2011 12:34:13 -0800 Subject: [PATCH 069/270] unit tests for issue 1560, customfiltersscore min and multiply search modes --- .../customscore/CustomScoreSearchTests.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.java index 8992e3ad4bc..1485240998f 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.java @@ -244,5 +244,45 @@ public class CustomScoreSearchTests extends AbstractNodesTests { assertThat(searchResponse.hits().getAt(1).id(), equalTo("1")); assertThat(searchResponse.hits().getAt(1).score(), equalTo(4.0f)); logger.info("--> Hit[1] {} Explanation {}", searchResponse.hits().getAt(1).id(), searchResponse.hits().getAt(1).explanation()); + + searchResponse = client.prepareSearch("test") + .setQuery(customFiltersScoreQuery(matchAllQuery()).scoreMode("min") + .add(termFilter("field", "value4"), 2) + .add(termFilter("field", "value1"), 3) + .add(termFilter("color", "red"), 5)) + .setExplain(true) + .execute().actionGet(); + + assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0)); + assertThat(searchResponse.hits().totalHits(), equalTo(4l)); + assertThat(searchResponse.hits().getAt(0).id(), equalTo("3")); + assertThat(searchResponse.hits().getAt(0).score(), equalTo(5.0f)); + logger.info("--> Hit[0] {} Explanation {}", searchResponse.hits().getAt(0).id(), searchResponse.hits().getAt(0).explanation()); + assertThat(searchResponse.hits().getAt(1).id(), equalTo("1")); + assertThat(searchResponse.hits().getAt(1).score(), equalTo(3.0f)); + assertThat(searchResponse.hits().getAt(2).id(), equalTo("4")); + assertThat(searchResponse.hits().getAt(2).score(), equalTo(2.0f)); + assertThat(searchResponse.hits().getAt(3).id(), equalTo("2")); + assertThat(searchResponse.hits().getAt(3).score(), equalTo(1.0f)); + + searchResponse = client.prepareSearch("test") + .setQuery(customFiltersScoreQuery(matchAllQuery()).scoreMode("multiply") + .add(termFilter("field", "value4"), 2) + .add(termFilter("field", "value1"), 3) + .add(termFilter("color", "red"), 5)) + .setExplain(true) + .execute().actionGet(); + + assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0)); + assertThat(searchResponse.hits().totalHits(), equalTo(4l)); + assertThat(searchResponse.hits().getAt(0).id(), equalTo("1")); + assertThat(searchResponse.hits().getAt(0).score(), equalTo(15.0f)); + logger.info("--> Hit[0] {} Explanation {}", searchResponse.hits().getAt(0).id(), searchResponse.hits().getAt(0).explanation()); + assertThat(searchResponse.hits().getAt(1).id(), equalTo("3")); + assertThat(searchResponse.hits().getAt(1).score(), equalTo(5.0f)); + assertThat(searchResponse.hits().getAt(2).id(), equalTo("4")); + assertThat(searchResponse.hits().getAt(2).score(), equalTo(2.0f)); + assertThat(searchResponse.hits().getAt(3).id(), equalTo("2")); + assertThat(searchResponse.hits().getAt(3).score(), equalTo(1.0f)); } } \ No newline at end of file From 73b74847aac3f7f4f3fcb61de11e54c6a643811e Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 22 Dec 2011 23:24:12 +0200 Subject: [PATCH 070/270] cleanup test --- .../customscore/CustomScoreSearchTests.java | 16 +++++++++------- .../customscore/CustomScoreSearchTests.yml | 6 ------ 2 files changed, 9 insertions(+), 13 deletions(-) delete mode 100644 src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.yml diff --git a/src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.java index 1485240998f..33b9a4b840d 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.java @@ -23,13 +23,14 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Client; import org.elasticsearch.test.integration.AbstractNodesTests; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import java.util.Arrays; import static org.elasticsearch.client.Requests.*; +import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.FilterBuilders.termFilter; import static org.elasticsearch.index.query.QueryBuilders.*; @@ -46,27 +47,27 @@ public class CustomScoreSearchTests extends AbstractNodesTests { private Client client; - @BeforeMethod + @BeforeClass public void createNodes() throws Exception { - startNode("server1"); + startNode("node1"); client = getClient(); } - @AfterMethod + @AfterClass public void closeNodes() { client.close(); closeAllNodes(); } protected Client getClient() { - return client("server1"); + return client("node1"); } @Test public void testCustomScriptBoost() throws Exception { client.admin().indices().prepareDelete().execute().actionGet(); + client.admin().indices().prepareCreate("test").setSettings(settingsBuilder().put("index.number_of_shards", 1)).execute().actionGet(); - client.admin().indices().create(createIndexRequest("test")).actionGet(); client.index(indexRequest("test").type("type1").id("1") .source(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject())).actionGet(); client.index(indexRequest("test").type("type1").id("2") @@ -152,6 +153,7 @@ public class CustomScoreSearchTests extends AbstractNodesTests { @Test public void testCustomFiltersScore() throws Exception { client.admin().indices().prepareDelete().execute().actionGet(); + client.admin().indices().prepareCreate("test").setSettings(settingsBuilder().put("index.number_of_shards", 1)).execute().actionGet(); client.prepareIndex("test", "type", "1").setSource("field", "value1", "color", "red").execute().actionGet(); client.prepareIndex("test", "type", "2").setSource("field", "value2", "color", "blue").execute().actionGet(); diff --git a/src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.yml b/src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.yml deleted file mode 100644 index e700f1f35fa..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.yml +++ /dev/null @@ -1,6 +0,0 @@ -cluster: - routing: - schedule: 100ms -index: - number_of_shards: 1 - number_of_replicas: 0 From 0ea58cacfd87b37cd70b360cdb0e6c3c603fe8df Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 24 Dec 2011 00:29:55 +0200 Subject: [PATCH 071/270] Query DSL: Add different execution models for terms filter, closes #1568. --- .../common/lucene/search/XBooleanFilter.java | 22 +++ .../index/query/TermsFilterBuilder.java | 17 ++- .../index/query/TermsFilterParser.java | 132 +++++++++++++++--- 3 files changed, 148 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/elasticsearch/common/lucene/search/XBooleanFilter.java b/src/main/java/org/elasticsearch/common/lucene/search/XBooleanFilter.java index 25c7654ffa6..168d9b19afb 100644 --- a/src/main/java/org/elasticsearch/common/lucene/search/XBooleanFilter.java +++ b/src/main/java/org/elasticsearch/common/lucene/search/XBooleanFilter.java @@ -145,6 +145,28 @@ public class XBooleanFilter extends Filter { } } + public void addMust(Filter filter) { + if (mustFilters == null) { + mustFilters = new ArrayList(); + } + mustFilters.add(filter); + } + + public void addShould(Filter filter) { + if (shouldFilters == null) { + shouldFilters = new ArrayList(); + } + shouldFilters.add(filter); + } + + public void addNot(Filter filter) { + if (notFilters == null) { + notFilters = new ArrayList(); + } + notFilters.add(filter); + } + + private boolean equalFilters(ArrayList filters1, ArrayList filters2) { return (filters1 == filters2) || ((filters1 != null) && filters1.equals(filters2)); diff --git a/src/main/java/org/elasticsearch/index/query/TermsFilterBuilder.java b/src/main/java/org/elasticsearch/index/query/TermsFilterBuilder.java index 2d69f44ceda..6ec30bfc89b 100644 --- a/src/main/java/org/elasticsearch/index/query/TermsFilterBuilder.java +++ b/src/main/java/org/elasticsearch/index/query/TermsFilterBuilder.java @@ -25,8 +25,6 @@ import java.io.IOException; /** * A filer for a field based on several terms matching on any of them. - * - * */ public class TermsFilterBuilder extends BaseFilterBuilder { @@ -39,6 +37,8 @@ public class TermsFilterBuilder extends BaseFilterBuilder { private String filterName; + private String execution; + /** * A filer for a field based on several terms matching on any of them. * @@ -116,6 +116,15 @@ public class TermsFilterBuilder extends BaseFilterBuilder { this.values = values; } + /** + * Sets the execution mode for the terms filter. Cane be either "plain", "bool" + * "and". Defaults to "plain". + */ + public TermsFilterBuilder execution(String execution) { + this.execution = execution; + return this; + } + /** * Sets the filter name for the filter that can be used when searching for matched_filters per hit. */ @@ -146,6 +155,10 @@ public class TermsFilterBuilder extends BaseFilterBuilder { } builder.endArray(); + if (execution != null) { + builder.field("execution", execution); + } + if (filterName != null) { builder.field("_name", filterName); } diff --git a/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java b/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java index 958d4eb2f8c..0bb50ec9d8b 100644 --- a/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java @@ -19,16 +19,21 @@ package org.elasticsearch.index.query; +import com.google.common.collect.Lists; import org.apache.lucene.index.Term; import org.apache.lucene.search.Filter; import org.apache.lucene.search.PublicTermsFilter; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.lucene.search.AndFilter; +import org.elasticsearch.common.lucene.search.TermFilter; +import org.elasticsearch.common.lucene.search.XBooleanFilter; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.cache.filter.support.CacheKeyFilter; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; +import java.util.List; import static org.elasticsearch.index.query.support.QueryParsers.wrapSmartNameFilter; @@ -53,40 +58,31 @@ public class TermsFilterParser implements FilterParser { XContentParser parser = parseContext.parser(); MapperService.SmartNameFieldMappers smartNameFieldMappers = null; - boolean cache = true; - PublicTermsFilter termsFilter = new PublicTermsFilter(); + Boolean cache = null; String filterName = null; String currentFieldName = null; CacheKeyFilter.Key cacheKey = null; XContentParser.Token token; + String execution = "plain"; + List terms = Lists.newArrayList(); + String fieldName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_ARRAY) { - String fieldName = currentFieldName; - FieldMapper fieldMapper = null; - smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); - if (smartNameFieldMappers != null) { - if (smartNameFieldMappers.hasMapper()) { - fieldMapper = smartNameFieldMappers.mapper(); - fieldName = fieldMapper.names().indexName(); - } - } + fieldName = currentFieldName; while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { String value = parser.text(); if (value == null) { throw new QueryParsingException(parseContext.index(), "No value specified for term filter"); } - if (fieldMapper != null) { - value = fieldMapper.indexedValue(value); - termsFilter.addTerm(fieldMapper.names().createIndexNameTerm(value)); - } else { - termsFilter.addTerm(new Term(fieldName, value)); - } + terms.add(value); } } else if (token.isValue()) { - if ("_name".equals(currentFieldName)) { + if ("execution".equals(currentFieldName)) { + execution = parser.text(); + } else if ("_name".equals(currentFieldName)) { filterName = parser.text(); } else if ("_cache".equals(currentFieldName)) { cache = parser.booleanValue(); @@ -96,9 +92,103 @@ public class TermsFilterParser implements FilterParser { } } - Filter filter = termsFilter; - if (cache) { - filter = parseContext.cacheFilter(filter, cacheKey); + if (fieldName == null) { + throw new QueryParsingException(parseContext.index(), "bool filter requires a field name, followed by array of terms"); + } + + FieldMapper fieldMapper = null; + smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); + if (smartNameFieldMappers != null) { + if (smartNameFieldMappers.hasMapper()) { + fieldMapper = smartNameFieldMappers.mapper(); + fieldName = fieldMapper.names().indexName(); + } + } + + + Filter filter; + if ("plain".equals(execution)) { + PublicTermsFilter termsFilter = new PublicTermsFilter(); + if (fieldMapper != null) { + for (String term : terms) { + termsFilter.addTerm(fieldMapper.names().createIndexNameTerm(fieldMapper.indexedValue(term))); + } + } else { + for (String term : terms) { + termsFilter.addTerm(new Term(fieldName, term)); + } + } + filter = termsFilter; + // cache the whole filter by default, or if explicitly told to + if (cache == null || cache) { + filter = parseContext.cacheFilter(filter, cacheKey); + } + } else if ("bool".equals(execution)) { + XBooleanFilter boolFiler = new XBooleanFilter(); + if (fieldMapper != null) { + for (String term : terms) { + boolFiler.addShould(parseContext.cacheFilter(fieldMapper.fieldFilter(term, parseContext), null)); + } + } else { + for (String term : terms) { + boolFiler.addShould(parseContext.cacheFilter(new TermFilter(new Term(fieldName, term)), null)); + } + } + filter = boolFiler; + // only cache if explicitly told to, since we cache inner filters + if (cache != null && cache) { + filter = parseContext.cacheFilter(filter, cacheKey); + } + } else if ("bool_nocache".equals(execution)) { + XBooleanFilter boolFiler = new XBooleanFilter(); + if (fieldMapper != null) { + for (String term : terms) { + boolFiler.addShould(fieldMapper.fieldFilter(term, parseContext)); + } + } else { + for (String term : terms) { + boolFiler.addShould(new TermFilter(new Term(fieldName, term))); + } + } + filter = boolFiler; + // cache the whole filter by default, or if explicitly told to + if (cache == null || cache) { + filter = parseContext.cacheFilter(filter, cacheKey); + } + } else if ("and".equals(execution)) { + List filters = Lists.newArrayList(); + if (fieldMapper != null) { + for (String term : terms) { + filters.add(parseContext.cacheFilter(fieldMapper.fieldFilter(term, parseContext), null)); + } + } else { + for (String term : terms) { + filters.add(parseContext.cacheFilter(new TermFilter(new Term(fieldName, term)), null)); + } + } + filter = new AndFilter(filters); + // only cache if explicitly told to, since we cache inner filters + if (cache != null && cache) { + filter = parseContext.cacheFilter(filter, cacheKey); + } + } else if ("and_nocache".equals(execution)) { + List filters = Lists.newArrayList(); + if (fieldMapper != null) { + for (String term : terms) { + filters.add(fieldMapper.fieldFilter(term, parseContext)); + } + } else { + for (String term : terms) { + filters.add(new TermFilter(new Term(fieldName, term))); + } + } + filter = new AndFilter(filters); + // cache the whole filter by default, or if explicitly told to + if (cache == null || cache) { + filter = parseContext.cacheFilter(filter, cacheKey); + } + } else { + throw new QueryParsingException(parseContext.index(), "bool filter execution value [" + execution + "] not supported"); } filter = wrapSmartNameFilter(filter, smartNameFieldMappers, parseContext); From aa078788f9eeb1c87d83805a5865e8424b2ef252 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 25 Dec 2011 13:33:02 +0200 Subject: [PATCH 072/270] Nested objects not deleted on "delete by query", closes #1537. --- .../index/mapper/MapperService.java | 4 + .../nested/IncludeAllChildrenQuery.java | 274 ++++++++++++++++++ .../shard/service/InternalIndexShard.java | 7 + .../integration/nested/SimpleNestedTests.java | 117 +++++++- 4 files changed, 400 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/elasticsearch/index/search/nested/IncludeAllChildrenQuery.java diff --git a/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 77228d16fa1..5ace3cb7a40 100644 --- a/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -140,6 +140,10 @@ public class MapperService extends AbstractIndexComponent implements Iterable iterator() { return mappers.values().iterator(); diff --git a/src/main/java/org/elasticsearch/index/search/nested/IncludeAllChildrenQuery.java b/src/main/java/org/elasticsearch/index/search/nested/IncludeAllChildrenQuery.java new file mode 100644 index 00000000000..85e3f38787f --- /dev/null +++ b/src/main/java/org/elasticsearch/index/search/nested/IncludeAllChildrenQuery.java @@ -0,0 +1,274 @@ +package org.elasticsearch.index.search.nested; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.*; +import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.common.lucene.docset.FixedBitDocSet; + +import java.io.IOException; +import java.util.Set; + +/** + * A special query that accepts a top level parent matching query, and returns all the children of that parent as + * well. This is handy when deleting by query. + */ +public class IncludeAllChildrenQuery extends Query { + + + private final Filter parentFilter; + private final Query parentQuery; + + // If we are rewritten, this is the original childQuery we + // were passed; we use this for .equals() and + // .hashCode(). This makes rewritten query equal the + // original, so that user does not have to .rewrite() their + // query before searching: + private final Query origParentQuery; + + + public IncludeAllChildrenQuery(Query parentQuery, Filter parentFilter) { + this.origParentQuery = parentQuery; + this.parentQuery = parentQuery; + this.parentFilter = parentFilter; + } + + IncludeAllChildrenQuery(Query origParentQuery, Query parentQuery, Filter parentFilter) { + this.origParentQuery = origParentQuery; + this.parentQuery = parentQuery; + this.parentFilter = parentFilter; + } + + @Override + public Weight createWeight(Searcher searcher) throws IOException { + return new IncludeAllChildrenWeight(parentQuery, parentQuery.createWeight(searcher), parentFilter); + } + + static class IncludeAllChildrenWeight extends Weight { + + private final Query parentQuery; + private final Weight parentWeight; + private final Filter parentsFilter; + + IncludeAllChildrenWeight(Query parentQuery, Weight parentWeight, Filter parentsFilter) { + this.parentQuery = parentQuery; + this.parentWeight = parentWeight; + this.parentsFilter = parentsFilter; + } + + @Override + public Query getQuery() { + return parentQuery; + } + + @Override + public float getValue() { + return parentWeight.getValue(); + } + + @Override + public float sumOfSquaredWeights() throws IOException { + return parentWeight.sumOfSquaredWeights() * parentQuery.getBoost() * parentQuery.getBoost(); + } + + @Override + public void normalize(float norm) { + parentWeight.normalize(norm * parentQuery.getBoost()); + } + + @Override + public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { + final Scorer parentScorer = parentWeight.scorer(reader, true, false); + + // no matches + if (parentScorer == null) { + return null; + } + + final int firstParentDoc = parentScorer.nextDoc(); + if (firstParentDoc == DocIdSetIterator.NO_MORE_DOCS) { + // No matches + return null; + } + + DocIdSet parents = parentsFilter.getDocIdSet(reader); + if (parents == null) { + // No matches + return null; + } + if (parents instanceof FixedBitDocSet) { + parents = ((FixedBitDocSet) parents).set(); + } + if (!(parents instanceof FixedBitSet)) { + throw new IllegalStateException("parentFilter must return OpenBitSet; got " + parents); + } + + + return new IncludeAllChildrenScorer(this, parentScorer, (FixedBitSet) parents, firstParentDoc); + } + + @Override + public Explanation explain(IndexReader reader, int doc) throws IOException { + return null; + } + + @Override + public boolean scoresDocsOutOfOrder() { + return false; + } + } + + static class IncludeAllChildrenScorer extends Scorer { + + private final Scorer parentScorer; + private final FixedBitSet parentBits; + + private int currentChildPointer = -1; + private int currentParentPointer = -1; + + private int currentDoc = -1; + + IncludeAllChildrenScorer(Weight weight, Scorer parentScorer, FixedBitSet parentBits, int currentParentPointer) { + super(weight); + this.parentScorer = parentScorer; + this.parentBits = parentBits; + this.currentParentPointer = currentParentPointer; + if (currentParentPointer == 0) { + currentChildPointer = 0; + } else { + this.currentChildPointer = parentBits.prevSetBit(currentParentPointer - 1); + if (currentChildPointer == -1) { + // no previous set parent, we delete from doc 0 + currentChildPointer = 0; + } else { + currentChildPointer++; // we only care about children + } + } + + currentDoc = currentChildPointer; + } + + @Override + protected void visitSubScorers(Query parent, BooleanClause.Occur relationship, ScorerVisitor visitor) { + super.visitSubScorers(parent, relationship, visitor); + parentScorer.visitScorers(visitor); + } + + @Override + public int nextDoc() throws IOException { + if (currentParentPointer == NO_MORE_DOCS) { + return (currentDoc = NO_MORE_DOCS); + } + + if (currentChildPointer == currentParentPointer) { + // we need to return the current parent as well, but prepare to return + // the next set of children + currentDoc = currentParentPointer; + currentParentPointer = parentScorer.nextDoc(); + if (currentParentPointer != NO_MORE_DOCS) { + currentChildPointer = parentBits.prevSetBit(currentParentPointer - 1); + if (currentChildPointer == -1) { + // no previous set parent, just set the child to the current parent + currentChildPointer = currentParentPointer; + } else { + currentChildPointer++; // we only care about children + } + } + } else { + currentDoc = currentChildPointer++; + } + + assert currentDoc != -1; + return currentDoc; + } + + @Override + public int advance(int target) throws IOException { + if (target == NO_MORE_DOCS) { + return (currentDoc = NO_MORE_DOCS); + } + + if (target == 0) { + return nextDoc(); + } + + currentParentPointer = parentScorer.advance(target); + if (currentParentPointer == NO_MORE_DOCS) { + return (currentDoc = NO_MORE_DOCS); + } + if (currentParentPointer == 0) { + currentChildPointer = 0; + } else { + currentChildPointer = parentBits.prevSetBit(currentParentPointer - 1); + if (currentChildPointer == -1) { + // no previous set parent, just set the child to 0 to delete all up to the parent + currentChildPointer = 0; + } else { + currentChildPointer++; // we only care about children + } + } + + currentDoc = currentChildPointer; + + return currentDoc; + } + + @Override + public float score() throws IOException { + return parentScorer.score(); + } + + @Override + public int docID() { + return currentDoc; + } + } + + @Override + public void extractTerms(Set terms) { + parentQuery.extractTerms(terms); + } + + @Override + public Query rewrite(IndexReader reader) throws IOException { + final Query parentRewrite = parentQuery.rewrite(reader); + if (parentRewrite != parentQuery) { + Query rewritten = new IncludeAllChildrenQuery(parentQuery, parentRewrite, parentFilter); + rewritten.setBoost(getBoost()); + return rewritten; + } else { + return this; + } + } + + @Override + public String toString(String field) { + return "IncludeAllChildrenQuery (" + parentQuery.toString() + ")"; + } + + @Override + public boolean equals(Object _other) { + if (_other instanceof IncludeAllChildrenQuery) { + final IncludeAllChildrenQuery other = (IncludeAllChildrenQuery) _other; + return origParentQuery.equals(other.origParentQuery) && + parentFilter.equals(other.parentFilter); + } else { + return false; + } + } + + @Override + public int hashCode() { + final int prime = 31; + int hash = 1; + hash = prime * hash + origParentQuery.hashCode(); + hash = prime * hash + parentFilter.hashCode(); + return hash; + } + + @Override + public Object clone() { + return new IncludeAllChildrenQuery((Query) origParentQuery.clone(), + parentFilter); + } +} diff --git a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java index 7a82e0d0673..ecca0b93068 100644 --- a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java +++ b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java @@ -52,6 +52,8 @@ import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.merge.scheduler.MergeSchedulerProvider; import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.refresh.RefreshStats; +import org.elasticsearch.index.search.nested.IncludeAllChildrenQuery; +import org.elasticsearch.index.search.nested.NonNestedDocsFilter; import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.search.stats.ShardSearchService; import org.elasticsearch.index.settings.IndexSettings; @@ -359,6 +361,11 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I @Override public void deleteByQuery(Engine.DeleteByQuery deleteByQuery) throws ElasticSearchException { writeAllowed(); + if (mapperService.hasNested()) { + // we need to wrap it to delete nested docs as well... + IncludeAllChildrenQuery nestedQuery = new IncludeAllChildrenQuery(deleteByQuery.query(), indexCache.filter().cache(NonNestedDocsFilter.INSTANCE)); + deleteByQuery = new Engine.DeleteByQuery(nestedQuery, deleteByQuery.source(), deleteByQuery.filteringAliases(), deleteByQuery.aliasFilter(), deleteByQuery.types()); + } if (logger.isTraceEnabled()) { logger.trace("delete_by_query [{}]", deleteByQuery.query()); } diff --git a/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java b/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java index d3a0c76cdd8..499700e4e75 100644 --- a/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java +++ b/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java @@ -25,7 +25,7 @@ import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.facet.FacetBuilders; import org.elasticsearch.search.facet.termsstats.TermsStatsFacet; import org.elasticsearch.test.integration.AbstractNodesTests; @@ -35,6 +35,7 @@ import org.testng.annotations.Test; import java.util.Arrays; +import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.FilterBuilders.nestedFilter; import static org.elasticsearch.index.query.QueryBuilders.*; @@ -178,6 +179,118 @@ public class SimpleNestedTests extends AbstractNodesTests { assertThat(searchResponse.hits().totalHits(), equalTo(1l)); } + @Test + public void simpleNestedDeletedByQuery1() throws Exception { + simpleNestedDeleteByQuery(3, 0); + } + + @Test + public void simpleNestedDeletedByQuery2() throws Exception { + simpleNestedDeleteByQuery(3, 1); + } + + @Test + public void simpleNestedDeletedByQuery3() throws Exception { + simpleNestedDeleteByQuery(3, 2); + } + + private void simpleNestedDeleteByQuery(int total, int docToDelete) throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + + client.admin().indices().prepareCreate("test") + .setSettings(settingsBuilder().put("index.number_of_shards", 1).put("index.referesh_interval", -1).build()) + .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") + .startObject("nested1") + .field("type", "nested") + .endObject() + .endObject().endObject().endObject()) + .execute().actionGet(); + + client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); + + + for (int i = 0; i < total; i++) { + client.prepareIndex("test", "type1", Integer.toString(i)).setSource(jsonBuilder().startObject() + .field("field1", "value1") + .startArray("nested1") + .startObject() + .field("n_field1", "n_value1_1") + .field("n_field2", "n_value2_1") + .endObject() + .startObject() + .field("n_field1", "n_value1_2") + .field("n_field2", "n_value2_2") + .endObject() + .endArray() + .endObject()).execute().actionGet(); + } + + + client.admin().indices().prepareFlush().setRefresh(true).execute().actionGet(); + IndicesStatusResponse statusResponse = client.admin().indices().prepareStatus().execute().actionGet(); + assertThat(statusResponse.index("test").docs().numDocs(), equalTo(total * 3)); + + client.prepareDeleteByQuery("test").setQuery(QueryBuilders.idsQuery("type1").ids(Integer.toString(docToDelete))).execute().actionGet(); + client.admin().indices().prepareFlush().setRefresh(true).execute().actionGet(); + statusResponse = client.admin().indices().prepareStatus().execute().actionGet(); + assertThat(statusResponse.index("test").docs().numDocs(), equalTo((total * 3) - 3)); + + for (int i = 0; i < total; i++) { + assertThat(client.prepareGet("test", "type1", Integer.toString(i)).execute().actionGet().exists(), equalTo(i != docToDelete)); + } + } + + @Test + public void noChildrenNestedDeletedByQuery1() throws Exception { + noChildrenNestedDeleteByQuery(3, 0); + } + + @Test + public void noChildrenNestedDeletedByQuery2() throws Exception { + noChildrenNestedDeleteByQuery(3, 1); + } + + @Test + public void noChildrenNestedDeletedByQuery3() throws Exception { + noChildrenNestedDeleteByQuery(3, 2); + } + + private void noChildrenNestedDeleteByQuery(int total, int docToDelete) throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + + client.admin().indices().prepareCreate("test") + .setSettings(settingsBuilder().put("index.number_of_shards", 1).put("index.referesh_interval", -1).build()) + .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") + .startObject("nested1") + .field("type", "nested") + .endObject() + .endObject().endObject().endObject()) + .execute().actionGet(); + + client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); + + + for (int i = 0; i < total; i++) { + client.prepareIndex("test", "type1", Integer.toString(i)).setSource(jsonBuilder().startObject() + .field("field1", "value1") + .endObject()).execute().actionGet(); + } + + + client.admin().indices().prepareFlush().setRefresh(true).execute().actionGet(); + IndicesStatusResponse statusResponse = client.admin().indices().prepareStatus().execute().actionGet(); + assertThat(statusResponse.index("test").docs().numDocs(), equalTo(total)); + + client.prepareDeleteByQuery("test").setQuery(QueryBuilders.idsQuery("type1").ids(Integer.toString(docToDelete))).execute().actionGet(); + client.admin().indices().prepareFlush().setRefresh(true).execute().actionGet(); + statusResponse = client.admin().indices().prepareStatus().execute().actionGet(); + assertThat(statusResponse.index("test").docs().numDocs(), equalTo((total) - 1)); + + for (int i = 0; i < total; i++) { + assertThat(client.prepareGet("test", "type1", Integer.toString(i)).execute().actionGet().exists(), equalTo(i != docToDelete)); + } + } + @Test public void multiNested() throws Exception { client.admin().indices().prepareDelete().execute().actionGet(); @@ -267,7 +380,7 @@ public class SimpleNestedTests extends AbstractNodesTests { client.admin().indices().prepareDelete().execute().actionGet(); client.admin().indices().prepareCreate("test") - .setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", numberOfShards)) + .setSettings(settingsBuilder().put("number_of_shards", numberOfShards)) .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") .startObject("nested1") .field("type", "nested").startObject("properties") From fb218babe30e79d8119f30ea25eed8e6af4dd74f Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 25 Dec 2011 17:37:58 +0200 Subject: [PATCH 073/270] add adapter streams that wrap another adapter and delegate all method calls, have handles use them --- .../common/io/stream/AdapterStreamInput.java | 119 +++++++++++++++ .../common/io/stream/AdapterStreamOutput.java | 136 ++++++++++++++++++ .../common/io/stream/HandlesStreamInput.java | 55 ++----- .../common/io/stream/HandlesStreamOutput.java | 40 +----- .../netty/MessageChannelHandler.java | 14 +- 5 files changed, 279 insertions(+), 85 deletions(-) create mode 100644 src/main/java/org/elasticsearch/common/io/stream/AdapterStreamInput.java create mode 100644 src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java diff --git a/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamInput.java new file mode 100644 index 00000000000..c5b61f33b6d --- /dev/null +++ b/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamInput.java @@ -0,0 +1,119 @@ +package org.elasticsearch.common.io.stream; + +import java.io.IOException; + +/** + */ +public abstract class AdapterStreamInput extends StreamInput { + + protected StreamInput in; + + protected AdapterStreamInput() { + } + + public AdapterStreamInput(StreamInput in) { + this.in = in; + } + + public void reset(StreamInput in) { + this.in = in; + } + + @Override + public byte readByte() throws IOException { + return in.readByte(); + } + + @Override + public void readBytes(byte[] b, int offset, int len) throws IOException { + in.readBytes(b, offset, len); + } + + @Override + public void reset() throws IOException { + in.reset(); + } + + @Override + public void close() throws IOException { + in.close(); + } + + @Override + public int read() throws IOException { + return in.read(); + } + + // override ones to direct them + + + @Override + public void readFully(byte[] b) throws IOException { + in.readFully(b); + } + + @Override + public short readShort() throws IOException { + return in.readShort(); + } + + @Override + public int readInt() throws IOException { + return in.readInt(); + } + + @Override + public int readVInt() throws IOException { + return in.readVInt(); + } + + @Override + public long readLong() throws IOException { + return in.readLong(); + } + + @Override + public long readVLong() throws IOException { + return in.readVLong(); + } + + @Override + public String readUTF() throws IOException { + return in.readUTF(); + } + + @Override + public int read(byte[] b) throws IOException { + return in.read(b); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return in.read(b, off, len); + } + + @Override + public long skip(long n) throws IOException { + return in.skip(n); + } + + @Override + public int available() throws IOException { + return in.available(); + } + + @Override + public void mark(int readlimit) { + in.mark(readlimit); + } + + @Override + public boolean markSupported() { + return in.markSupported(); + } + + @Override + public String toString() { + return in.toString(); + } +} diff --git a/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java b/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java new file mode 100644 index 00000000000..2010160407a --- /dev/null +++ b/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java @@ -0,0 +1,136 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.io.stream; + +import java.io.IOException; + +/** + */ +public class AdapterStreamOutput extends StreamOutput { + + protected StreamOutput out; + + public AdapterStreamOutput(StreamOutput out) { + this.out = out; + } + + public void reset(StreamOutput out) throws IOException { + this.out = out; + } + + public StreamOutput wrappedOut() { + return this.out; + } + + @Override + public void writeByte(byte b) throws IOException { + out.writeByte(b); + } + + @Override + public void writeBytes(byte[] b, int offset, int length) throws IOException { + out.writeBytes(b, offset, length); + } + + @Override + public void flush() throws IOException { + out.flush(); + } + + @Override + public void close() throws IOException { + out.close(); + } + + @Override + public void reset() throws IOException { + out.reset(); + } + + @Override + public void writeBytes(byte[] b) throws IOException { + out.writeBytes(b); + } + + @Override + public void writeBytes(byte[] b, int length) throws IOException { + out.writeBytes(b, length); + } + + @Override + public void writeInt(int i) throws IOException { + out.writeInt(i); + } + + @Override + public void writeVInt(int i) throws IOException { + out.writeVInt(i); + } + + @Override + public void writeLong(long i) throws IOException { + out.writeLong(i); + } + + @Override + public void writeVLong(long i) throws IOException { + out.writeVLong(i); + } + + @Override + public void writeUTF(String str) throws IOException { + out.writeUTF(str); + } + + @Override + public void writeFloat(float v) throws IOException { + out.writeFloat(v); + } + + @Override + public void writeDouble(double v) throws IOException { + out.writeDouble(v); + } + + @Override + public void writeBoolean(boolean b) throws IOException { + out.writeBoolean(b); + } + + @Override + public void write(int b) throws IOException { + out.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + } + + @Override + public void write(byte[] b) throws IOException { + out.write(b); + } + + @Override + public String toString() { + return out.toString(); + } +} diff --git a/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamInput.java index 35f63ec72e8..8551dedd70f 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamInput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamInput.java @@ -26,19 +26,18 @@ import java.io.IOException; /** * */ -public class HandlesStreamInput extends StreamInput { - - private StreamInput in; +public class HandlesStreamInput extends AdapterStreamInput { private final TIntObjectHashMap handles = new TIntObjectHashMap(); private final TIntObjectHashMap identityHandles = new TIntObjectHashMap(); HandlesStreamInput() { + super(); } public HandlesStreamInput(StreamInput in) { - this.in = in; + super(in); } @Override @@ -65,49 +64,21 @@ public class HandlesStreamInput extends StreamInput { } } - @Override - public int read() throws IOException { - return in.read(); - } - - @Override - public int read(byte[] b) throws IOException { - return in.read(b); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - return in.read(b, off, len); - } - - @Override - public byte readByte() throws IOException { - return in.readByte(); - } - - @Override - public void readBytes(byte[] b, int offset, int len) throws IOException { - in.readBytes(b, offset, len); - } - - public void cleanHandles() { - handles.clear(); - } - @Override public void reset() throws IOException { - in.reset(); - handles.clear(); - } - - public void reset(StreamInput in) { - this.in = in; + super.reset(); handles.clear(); identityHandles.clear(); } - @Override - public void close() throws IOException { - in.close(); + public void reset(StreamInput in) { + super.reset(in); + handles.clear(); + identityHandles.clear(); + } + + public void cleanHandles() { + handles.clear(); + identityHandles.clear(); } } diff --git a/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamOutput.java b/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamOutput.java index fa1be3eac93..2ad85045109 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamOutput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamOutput.java @@ -28,15 +28,13 @@ import java.util.Arrays; /** * */ -public class HandlesStreamOutput extends StreamOutput { +public class HandlesStreamOutput extends AdapterStreamOutput { - private static final int DEFAULT_IDENTITY_THRESHOLD = 100; + private static final int DEFAULT_IDENTITY_THRESHOLD = 50; // a threshold above which strings will use identity check private final int identityThreshold; - private StreamOutput out; - private final TObjectIntHashMap handles = new TObjectIntHashMap(Constants.DEFAULT_CAPACITY, Constants.DEFAULT_LOAD_FACTOR, -1); private final HandleTable identityHandles = new HandleTable(10, (float) 3.00); @@ -46,7 +44,7 @@ public class HandlesStreamOutput extends StreamOutput { } public HandlesStreamOutput(StreamOutput out, int identityThreshold) { - this.out = out; + super(out); this.identityThreshold = identityThreshold; } @@ -78,21 +76,6 @@ public class HandlesStreamOutput extends StreamOutput { } } - @Override - public void writeByte(byte b) throws IOException { - out.writeByte(b); - } - - @Override - public void writeBytes(byte[] b, int offset, int length) throws IOException { - out.writeBytes(b, offset, length); - } - - public void cleanHandles() { - handles.clear(); - identityHandles.clear(); - } - @Override public void reset() throws IOException { handles.clear(); @@ -101,25 +84,10 @@ public class HandlesStreamOutput extends StreamOutput { } public void reset(StreamOutput out) throws IOException { - this.out = out; + super.reset(out); reset(); } - @Override - public void flush() throws IOException { - out.flush(); - } - - @Override - public void close() throws IOException { - out.close(); - } - - public StreamOutput wrappedOut() { - return this.out; - } - - /** * Lightweight identity hash table which maps objects to integer handles, * assigned in ascending order. diff --git a/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java b/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java index 71ebf872a72..7b98998e8da 100644 --- a/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java +++ b/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java @@ -186,15 +186,15 @@ public class MessageChannelHandler extends SimpleChannelUpstreamHandler { byte status = buffer.readByte(); boolean isRequest = TransportStreams.statusIsRequest(status); - HandlesStreamInput handlesStream; + HandlesStreamInput wrappedStream; if (TransportStreams.statusIsCompress(status)) { - handlesStream = CachedStreamInput.cachedHandlesLzf(streamIn); + wrappedStream = CachedStreamInput.cachedHandlesLzf(streamIn); } else { - handlesStream = CachedStreamInput.cachedHandles(streamIn); + wrappedStream = CachedStreamInput.cachedHandles(streamIn); } if (isRequest) { - String action = handleRequest(channel, handlesStream, requestId); + String action = handleRequest(channel, wrappedStream, requestId); if (buffer.readerIndex() != expectedIndexReader) { if (buffer.readerIndex() < expectedIndexReader) { logger.warn("Message not fully read (request) for [{}] and action [{}], resetting", requestId, action); @@ -208,9 +208,9 @@ public class MessageChannelHandler extends SimpleChannelUpstreamHandler { // ignore if its null, the adapter logs it if (handler != null) { if (TransportStreams.statusIsError(status)) { - handlerResponseError(handlesStream, handler); + handlerResponseError(wrappedStream, handler); } else { - handleResponse(handlesStream, handler); + handleResponse(wrappedStream, handler); } } else { // if its null, skip those bytes @@ -225,7 +225,7 @@ public class MessageChannelHandler extends SimpleChannelUpstreamHandler { buffer.readerIndex(expectedIndexReader); } } - handlesStream.cleanHandles(); + wrappedStream.cleanHandles(); } private void handleResponse(StreamInput buffer, final TransportResponseHandler handler) { From 038d91d13400fa50e3c509b4a22f79df7feda8db Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 25 Dec 2011 18:37:42 +0200 Subject: [PATCH 074/270] Java API: ScriptSortBuilder does not pass the lang used, closes #1569. --- .../org/elasticsearch/search/sort/ScriptSortBuilder.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java index ef96fc8c339..542903db5a7 100644 --- a/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java +++ b/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java @@ -27,8 +27,6 @@ import java.util.Map; /** * Script sort builder allows to sort based on a custom script expression. - * - * */ public class ScriptSortBuilder extends SortBuilder { @@ -100,6 +98,9 @@ public class ScriptSortBuilder extends SortBuilder { if (order == SortOrder.DESC) { builder.field("reverse", true); } + if (lang != null) { + builder.field("lang", lang); + } if (this.params != null) { builder.field("params", this.params); } From 37398c2000180fb432c17570a78fc838211e72d1 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 26 Dec 2011 16:32:30 +0200 Subject: [PATCH 075/270] ignore index shard missing failure when refreshing as it might happen --- .../admin/indices/refresh/TransportRefreshAction.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java b/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java index f91699f3ffc..6ef2fd69fe4 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java @@ -32,6 +32,7 @@ import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexShardMissingException; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.IllegalIndexShardStateException; import org.elasticsearch.index.shard.service.IndexShard; @@ -47,8 +48,6 @@ import static com.google.common.collect.Lists.newArrayList; /** * Refresh action. - * - * */ public class TransportRefreshAction extends TransportBroadcastOperationAction { @@ -95,6 +94,9 @@ public class TransportRefreshAction extends TransportBroadcastOperationAction Date: Mon, 26 Dec 2011 16:49:55 +0200 Subject: [PATCH 076/270] Search: Support partial fields that can returns partial view of the _source, closes #1570. --- .../action/search/SearchRequestBuilder.java | 27 +++ .../xcontent/support/XContentMapValues.java | 4 +- .../elasticsearch/search/SearchHitField.java | 5 +- .../elasticsearch/search/SearchModule.java | 2 + .../search/builder/SearchSourceBuilder.java | 89 +++++++- .../search/fetch/FetchPhase.java | 25 ++- .../fetch/partial/PartialFieldsContext.java | 67 ++++++ .../partial/PartialFieldsFetchSubPhase.java | 84 ++++++++ .../partial/PartialFieldsParseElement.java | 95 +++++++++ .../search/internal/SearchContext.java | 16 +- .../search/lookup/SourceLookup.java | 4 + .../search/fields/SearchFieldsTests.java | 173 +++++++++++++++- .../scriptfield/ScriptFieldSearchTests.java | 192 ------------------ .../support/XContentMapValuesTests.java | 4 + 14 files changed, 574 insertions(+), 213 deletions(-) create mode 100644 src/main/java/org/elasticsearch/search/fetch/partial/PartialFieldsContext.java create mode 100644 src/main/java/org/elasticsearch/search/fetch/partial/PartialFieldsFetchSubPhase.java create mode 100644 src/main/java/org/elasticsearch/search/fetch/partial/PartialFieldsParseElement.java delete mode 100644 src/test/java/org/elasticsearch/test/integration/search/scriptfield/ScriptFieldSearchTests.java diff --git a/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java b/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java index 294e4f33b6f..ca85f603bb6 100644 --- a/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java +++ b/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Client; import org.elasticsearch.client.action.support.BaseRequestBuilder; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.FilterBuilder; @@ -392,6 +393,32 @@ public class SearchRequestBuilder extends BaseRequestBuilder 0) { boolean atLeastOnOneIncludeMatched = false; for (String include : includes) { - if (Regex.simpleMatch(include, path)) { + // check for prefix as well, something like: obj1.arr1.* + // note, this does not work well with middle matches, like obj1.*.obj3 + if (include.startsWith(path) || Regex.simpleMatch(include, path)) { atLeastOnOneIncludeMatched = true; break; } diff --git a/src/main/java/org/elasticsearch/search/SearchHitField.java b/src/main/java/org/elasticsearch/search/SearchHitField.java index 773e6c0b7ad..69a886d2b39 100644 --- a/src/main/java/org/elasticsearch/search/SearchHitField.java +++ b/src/main/java/org/elasticsearch/search/SearchHitField.java @@ -26,7 +26,6 @@ import java.util.List; /** * A single field name and values part of a {@link SearchHit}. * - * * @see SearchHit */ public interface SearchHitField extends Streamable, Iterable { @@ -44,12 +43,12 @@ public interface SearchHitField extends Streamable, Iterable { /** * The first value of the hit. */ - Object value(); + V value(); /** * The first value of the hit. */ - Object getValue(); + V getValue(); /** * The field values. diff --git a/src/main/java/org/elasticsearch/search/SearchModule.java b/src/main/java/org/elasticsearch/search/SearchModule.java index 6666c21a87d..53d8acd4741 100644 --- a/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/src/main/java/org/elasticsearch/search/SearchModule.java @@ -30,6 +30,7 @@ import org.elasticsearch.search.facet.FacetModule; import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.fetch.explain.ExplainFetchSubPhase; import org.elasticsearch.search.fetch.matchedfilters.MatchedFiltersFetchSubPhase; +import org.elasticsearch.search.fetch.partial.PartialFieldsFetchSubPhase; import org.elasticsearch.search.fetch.script.ScriptFieldsFetchSubPhase; import org.elasticsearch.search.fetch.version.VersionFetchSubPhase; import org.elasticsearch.search.highlight.HighlightPhase; @@ -55,6 +56,7 @@ public class SearchModule extends AbstractModule implements SpawnModules { bind(FetchPhase.class).asEagerSingleton(); bind(ExplainFetchSubPhase.class).asEagerSingleton(); bind(ScriptFieldsFetchSubPhase.class).asEagerSingleton(); + bind(PartialFieldsFetchSubPhase.class).asEagerSingleton(); bind(VersionFetchSubPhase.class).asEagerSingleton(); bind(MatchedFiltersFetchSubPhase.class).asEagerSingleton(); bind(HighlightPhase.class).asEagerSingleton(); diff --git a/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 1b9fd6e77fd..a301b5ec63c 100644 --- a/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -25,6 +25,7 @@ import gnu.trove.iterator.TObjectFloatIterator; import gnu.trove.map.hash.TObjectFloatHashMap; import org.elasticsearch.ElasticSearchGenerationException; import org.elasticsearch.client.Requests; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Unicode; import org.elasticsearch.common.io.BytesStream; import org.elasticsearch.common.xcontent.ToXContent; @@ -48,7 +49,6 @@ import java.util.Map; * A search source builder allowing to easily build search source. Simple construction * using {@link org.elasticsearch.search.builder.SearchSourceBuilder#searchSource()}. * - * * @see org.elasticsearch.action.search.SearchRequest#source(SearchSourceBuilder) */ public class SearchSourceBuilder implements ToXContent { @@ -94,8 +94,8 @@ public class SearchSourceBuilder implements ToXContent { private Float minScore; private List fieldNames; - private List scriptFields; + private List partialFields; private List facets; @@ -463,6 +463,38 @@ public class SearchSourceBuilder implements ToXContent { return this; } + /** + * Adds a partial field based on _source, with an "include" and/or "exclude" set which can include simple wildcard + * elements. + * + * @param name The name of the field + * @param include An optional include (optionally wildcarded) pattern from _source + * @param exclude An optional exclude (optionally wildcarded) pattern from _source + */ + public SearchSourceBuilder partialField(String name, @Nullable String include, @Nullable String exclude) { + if (partialFields == null) { + partialFields = Lists.newArrayList(); + } + partialFields.add(new PartialField(name, include, exclude)); + return this; + } + + /** + * Adds a partial field based on _source, with an "includes" and/or "excludes set which can include simple wildcard + * elements. + * + * @param name The name of the field + * @param includes An optional list of includes (optionally wildcarded) patterns from _source + * @param excludes An optional list of excludes (optionally wildcarded) patterns from _source + */ + public SearchSourceBuilder partialField(String name, @Nullable String[] includes, @Nullable String[] excludes) { + if (partialFields == null) { + partialFields = Lists.newArrayList(); + } + partialFields.add(new PartialField(name, includes, excludes)); + return this; + } + /** * Sets the boost a specific index will receive when the query is executeed against it. * @@ -582,6 +614,29 @@ public class SearchSourceBuilder implements ToXContent { } } + if (partialFields != null) { + builder.startObject("partial_fields"); + for (PartialField partialField : partialFields) { + builder.startObject(partialField.name()); + if (partialField.includes() != null) { + if (partialField.includes().length == 1) { + builder.field("include", partialField.includes()[0]); + } else { + builder.field("include", partialField.includes()); + } + } + if (partialField.excludes() != null) { + if (partialField.excludes().length == 1) { + builder.field("exclude", partialField.excludes()[0]); + } else { + builder.field("exclude", partialField.excludes()); + } + } + builder.endObject(); + } + builder.endObject(); + } + if (scriptFields != null) { builder.startObject("script_fields"); for (ScriptField scriptField : scriptFields) { @@ -683,4 +738,34 @@ public class SearchSourceBuilder implements ToXContent { return params; } } + + private static class PartialField { + private final String name; + private final String[] includes; + private final String[] excludes; + + private PartialField(String name, String[] includes, String[] excludes) { + this.name = name; + this.includes = includes; + this.excludes = excludes; + } + + private PartialField(String name, String include, String exclude) { + this.name = name; + this.includes = include == null ? null : new String[]{include}; + this.excludes = exclude == null ? null : new String[]{exclude}; + } + + public String name() { + return name; + } + + public String[] includes() { + return includes; + } + + public String[] excludes() { + return excludes; + } + } } diff --git a/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 620428ef7a3..997194a9d8a 100644 --- a/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -44,6 +44,7 @@ import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.SearchPhase; import org.elasticsearch.search.fetch.explain.ExplainFetchSubPhase; import org.elasticsearch.search.fetch.matchedfilters.MatchedFiltersFetchSubPhase; +import org.elasticsearch.search.fetch.partial.PartialFieldsFetchSubPhase; import org.elasticsearch.search.fetch.script.ScriptFieldsFetchSubPhase; import org.elasticsearch.search.fetch.version.VersionFetchSubPhase; import org.elasticsearch.search.highlight.HighlightPhase; @@ -66,9 +67,9 @@ public class FetchPhase implements SearchPhase { private final FetchSubPhase[] fetchSubPhases; @Inject - public FetchPhase(HighlightPhase highlightPhase, ScriptFieldsFetchSubPhase scriptFieldsPhase, + public FetchPhase(HighlightPhase highlightPhase, ScriptFieldsFetchSubPhase scriptFieldsPhase, PartialFieldsFetchSubPhase partialFieldsPhase, MatchedFiltersFetchSubPhase matchFiltersPhase, ExplainFetchSubPhase explainPhase, VersionFetchSubPhase versionPhase) { - this.fetchSubPhases = new FetchSubPhase[]{scriptFieldsPhase, matchFiltersPhase, explainPhase, highlightPhase, versionPhase}; + this.fetchSubPhases = new FetchSubPhase[]{scriptFieldsPhase, partialFieldsPhase, matchFiltersPhase, explainPhase, highlightPhase, versionPhase}; } @Override @@ -89,13 +90,19 @@ public class FetchPhase implements SearchPhase { ResetFieldSelector fieldSelector; List extractFieldNames = null; boolean sourceRequested = false; - if (context.hasScriptFields() && !context.hasFieldNames()) { - // we ask for script fields, and no field names, don't load the source - fieldSelector = UidFieldSelector.INSTANCE; - sourceRequested = false; - } else if (!context.hasFieldNames()) { - fieldSelector = new UidAndSourceFieldSelector(); - sourceRequested = true; + if (!context.hasFieldNames()) { + if (context.hasPartialFields()) { + // partial fields need the source, so fetch it, but don't return it + fieldSelector = new UidAndSourceFieldSelector(); + sourceRequested = false; + } else if (context.hasScriptFields()) { + // we ask for script fields, and no field names, don't load the source + fieldSelector = UidFieldSelector.INSTANCE; + sourceRequested = false; + } else { + fieldSelector = new UidAndSourceFieldSelector(); + sourceRequested = true; + } } else if (context.fieldNames().isEmpty()) { fieldSelector = UidFieldSelector.INSTANCE; sourceRequested = false; diff --git a/src/main/java/org/elasticsearch/search/fetch/partial/PartialFieldsContext.java b/src/main/java/org/elasticsearch/search/fetch/partial/PartialFieldsContext.java new file mode 100644 index 00000000000..ef211778953 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/fetch/partial/PartialFieldsContext.java @@ -0,0 +1,67 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.fetch.partial; + +import com.google.common.collect.Lists; + +import java.util.List; + +/** + */ +public class PartialFieldsContext { + + public static class PartialField { + private final String name; + private final String[] includes; + private final String[] excludes; + + public PartialField(String name, String[] includes, String[] excludes) { + this.name = name; + this.includes = includes; + this.excludes = excludes; + } + + public String name() { + return this.name; + } + + public String[] includes() { + return this.includes; + } + + public String[] excludes() { + return this.excludes; + } + } + + private final List fields = Lists.newArrayList(); + + public PartialFieldsContext() { + + } + + public void add(PartialField field) { + fields.add(field); + } + + public List fields() { + return this.fields; + } +} diff --git a/src/main/java/org/elasticsearch/search/fetch/partial/PartialFieldsFetchSubPhase.java b/src/main/java/org/elasticsearch/search/fetch/partial/PartialFieldsFetchSubPhase.java new file mode 100644 index 00000000000..772edb8d335 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/fetch/partial/PartialFieldsFetchSubPhase.java @@ -0,0 +1,84 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.fetch.partial; + +import com.google.common.collect.ImmutableMap; +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.search.SearchParseElement; +import org.elasticsearch.search.fetch.FetchSubPhase; +import org.elasticsearch.search.internal.InternalSearchHit; +import org.elasticsearch.search.internal.InternalSearchHitField; +import org.elasticsearch.search.internal.SearchContext; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +/** + */ +public class PartialFieldsFetchSubPhase implements FetchSubPhase { + + @Inject + public PartialFieldsFetchSubPhase() { + + } + + @Override + public Map parseElements() { + ImmutableMap.Builder parseElements = ImmutableMap.builder(); + parseElements.put("partial_fields", new PartialFieldsParseElement()) + .put("partialFields", new PartialFieldsParseElement()); + return parseElements.build(); + } + + @Override + public boolean hitsExecutionNeeded(SearchContext context) { + return false; + } + + @Override + public void hitsExecute(SearchContext context, InternalSearchHit[] hits) throws ElasticSearchException { + } + + @Override + public boolean hitExecutionNeeded(SearchContext context) { + return context.hasPartialFields(); + } + + @Override + public void hitExecute(SearchContext context, HitContext hitContext) throws ElasticSearchException { + for (PartialFieldsContext.PartialField field : context.partialFields().fields()) { + Object value = context.lookup().source().filter(field.includes(), field.excludes()); + + if (hitContext.hit().fieldsOrNull() == null) { + hitContext.hit().fields(new HashMap(2)); + } + + SearchHitField hitField = hitContext.hit().fields().get(field.name()); + if (hitField == null) { + hitField = new InternalSearchHitField(field.name(), new ArrayList(2)); + hitContext.hit().fields().put(field.name(), hitField); + } + hitField.values().add(value); + } + } +} diff --git a/src/main/java/org/elasticsearch/search/fetch/partial/PartialFieldsParseElement.java b/src/main/java/org/elasticsearch/search/fetch/partial/PartialFieldsParseElement.java new file mode 100644 index 00000000000..34ebefb7161 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/fetch/partial/PartialFieldsParseElement.java @@ -0,0 +1,95 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.fetch.partial; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchParseElement; +import org.elasticsearch.search.internal.SearchContext; + +import java.util.ArrayList; +import java.util.List; + +/** + * 
+ * "partial_fields" : {
+ *  "test1" : {
+ *      "includes" : "doc['field_name'].value"
+ *  },
+ *  "test2" : {
+ *      "excludes" : "..."
+ *  }
+ * }
+ *
+ */ +public class PartialFieldsParseElement implements SearchParseElement { + + @Override + public void parse(XContentParser parser, SearchContext context) throws Exception { + XContentParser.Token token; + String currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + String fieldName = currentFieldName; + List includes = null; + List excludes = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_ARRAY) { + if ("includes".equals(currentFieldName) || "include".equals(currentFieldName)) { + if (includes == null) { + includes = new ArrayList(2); + } + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + includes.add(parser.text()); + } + } else if ("excludes".equals(currentFieldName) || "exclude".equals(currentFieldName)) { + if (excludes == null) { + excludes = new ArrayList(2); + } + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + excludes.add(parser.text()); + } + } + } else if (token.isValue()) { + if ("include".equals(currentFieldName)) { + if (includes == null) { + includes = new ArrayList(2); + } + includes.add(parser.text()); + } else if ("exclude".equals(currentFieldName)) { + if (excludes == null) { + excludes = new ArrayList(2); + } + excludes.add(parser.text()); + } + } + } + PartialFieldsContext.PartialField field = new PartialFieldsContext.PartialField(fieldName, + includes == null ? Strings.EMPTY_ARRAY : includes.toArray(new String[includes.size()]), + excludes == null ? Strings.EMPTY_ARRAY : excludes.toArray(new String[excludes.size()])); + context.partialFields().add(field); + } + } + } +} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/src/main/java/org/elasticsearch/search/internal/SearchContext.java index cd55ab5bba6..36847015558 100644 --- a/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -50,6 +50,7 @@ import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.dfs.DfsSearchResult; import org.elasticsearch.search.facet.SearchContextFacets; import org.elasticsearch.search.fetch.FetchSearchResult; +import org.elasticsearch.search.fetch.partial.PartialFieldsContext; import org.elasticsearch.search.fetch.script.ScriptFieldsContext; import org.elasticsearch.search.highlight.SearchContextHighlight; import org.elasticsearch.search.lookup.SearchLookup; @@ -117,6 +118,8 @@ public class SearchContext implements Releasable { private boolean version = false; // by default, we don't return versions private List fieldNames; + private ScriptFieldsContext scriptFields; + private PartialFieldsContext partialFields; private int from = -1; @@ -146,8 +149,6 @@ public class SearchContext implements Releasable { private SearchContextHighlight highlight; - private ScriptFieldsContext scriptFields; - private SearchLookup searchLookup; private boolean queryRewritten; @@ -278,6 +279,17 @@ public class SearchContext implements Releasable { return this.scriptFields; } + public boolean hasPartialFields() { + return partialFields != null; + } + + public PartialFieldsContext partialFields() { + if (partialFields == null) { + partialFields = new PartialFieldsContext(); + } + return this.partialFields; + } + public ContextIndexSearcher searcher() { return this.searcher; } diff --git a/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index 655c5c74b45..125cc6a6149 100644 --- a/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -116,6 +116,10 @@ public class SourceLookup implements Map { return XContentMapValues.extractRawValues(path, loadSourceIfNeeded()); } + public Object filter(String[] includes, String[] excludes) { + return XContentMapValues.filter(loadSourceIfNeeded(), includes, excludes); + } + public Object extractValue(String path) { return XContentMapValues.extractValue(path, loadSourceIfNeeded()); } diff --git a/src/test/java/org/elasticsearch/test/integration/search/fields/SearchFieldsTests.java b/src/test/java/org/elasticsearch/test/integration/search/fields/SearchFieldsTests.java index 56abdb0c181..2e2beb64b3b 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/fields/SearchFieldsTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/fields/SearchFieldsTests.java @@ -21,12 +21,20 @@ package org.elasticsearch.test.integration.search.fields; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.integration.AbstractNodesTests; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.client.Requests.refreshRequest; +import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.hamcrest.MatcherAssert.assertThat; @@ -41,8 +49,8 @@ public class SearchFieldsTests extends AbstractNodesTests { @BeforeClass public void createNodes() throws Exception { - startNode("server1"); - startNode("server2"); + startNode("node1", settingsBuilder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)); + startNode("client1", settingsBuilder().put("node.client", true).build()); client = getClient(); } @@ -53,13 +61,14 @@ public class SearchFieldsTests extends AbstractNodesTests { } protected Client getClient() { - return client("server1"); + return client("client1"); } @Test public void testStoredFields() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); client.admin().indices().prepareCreate("test").execute().actionGet(); - client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); + client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("field1").field("type", "string").field("store", "yes").endObject() @@ -112,4 +121,160 @@ public class SearchFieldsTests extends AbstractNodesTests { assertThat(searchResponse.hits().getAt(0).fields().get("field1").value().toString(), equalTo("value1")); assertThat(searchResponse.hits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); } + + @Test + public void testScriptDocAndFields() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + client.admin().indices().prepareCreate("test").execute().actionGet(); + client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); + + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("num1").field("type", "double").field("store", "yes").endObject() + .endObject().endObject().endObject().string(); + + client.admin().indices().preparePutMapping().setType("type1").setSource(mapping).execute().actionGet(); + + client.prepareIndex("test", "type1", "1") + .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).field("date", "1970-01-01T00:00:00").endObject()) + .execute().actionGet(); + client.admin().indices().prepareFlush().execute().actionGet(); + client.prepareIndex("test", "type1", "2") + .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).field("date", "1970-01-01T00:00:25").endObject()) + .execute().actionGet(); + client.admin().indices().prepareFlush().execute().actionGet(); + client.prepareIndex("test", "type1", "3") + .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).field("date", "1970-01-01T00:02:00").endObject()) + .execute().actionGet(); + client.admin().indices().refresh(refreshRequest()).actionGet(); + + logger.info("running doc['num1'].value"); + SearchResponse response = client.prepareSearch() + .setQuery(matchAllQuery()) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", "doc['num1'].value") + .addScriptField("sNum1_field", "_fields['num1'].value") + .addScriptField("date1", "doc['date'].date.millis") + .execute().actionGet(); + + assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0)); + + assertThat(response.hits().totalHits(), equalTo(3l)); + assertThat(response.hits().getAt(0).isSourceEmpty(), equalTo(true)); + assertThat(response.hits().getAt(0).id(), equalTo("1")); + assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0)); + assertThat((Double) response.hits().getAt(0).fields().get("sNum1_field").values().get(0), equalTo(1.0)); + assertThat((Long) response.hits().getAt(0).fields().get("date1").values().get(0), equalTo(0l)); + assertThat(response.hits().getAt(1).id(), equalTo("2")); + assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0)); + assertThat((Double) response.hits().getAt(1).fields().get("sNum1_field").values().get(0), equalTo(2.0)); + assertThat((Long) response.hits().getAt(1).fields().get("date1").values().get(0), equalTo(25000l)); + assertThat(response.hits().getAt(2).id(), equalTo("3")); + assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0)); + assertThat((Double) response.hits().getAt(2).fields().get("sNum1_field").values().get(0), equalTo(3.0)); + assertThat((Long) response.hits().getAt(2).fields().get("date1").values().get(0), equalTo(120000l)); + + logger.info("running doc['num1'].value * factor"); + Map params = MapBuilder.newMapBuilder().put("factor", 2.0).map(); + response = client.prepareSearch() + .setQuery(matchAllQuery()) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", "doc['num1'].value * factor", params) + .execute().actionGet(); + + assertThat(response.hits().totalHits(), equalTo(3l)); + assertThat(response.hits().getAt(0).id(), equalTo("1")); + assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0)); + assertThat(response.hits().getAt(1).id(), equalTo("2")); + assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(4.0)); + assertThat(response.hits().getAt(2).id(), equalTo("3")); + assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(6.0)); + } + + @Test + public void testScriptFieldUsingSource() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + client.admin().indices().prepareCreate("test").execute().actionGet(); + client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); + + client.prepareIndex("test", "type1", "1") + .setSource(jsonBuilder().startObject() + .startObject("obj1").field("test", "something").endObject() + .startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject() + .startArray("arr3").startObject().field("arr3_field1", "arr3_value1").endObject().endArray() + .endObject()) + .execute().actionGet(); + client.admin().indices().refresh(refreshRequest()).actionGet(); + + SearchResponse response = client.prepareSearch() + .setQuery(matchAllQuery()) + .addField("_source.obj1") // we also automatically detect _source in fields + .addScriptField("s_obj1", "_source.obj1") + .addScriptField("s_obj1_test", "_source.obj1.test") + .addScriptField("s_obj2", "_source.obj2") + .addScriptField("s_obj2_arr2", "_source.obj2.arr2") + .addScriptField("s_arr3", "_source.arr3") + .execute().actionGet(); + + assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0)); + + Map sObj1 = response.hits().getAt(0).field("_source.obj1").value(); + assertThat(sObj1.get("test").toString(), equalTo("something")); + assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something")); + + sObj1 = response.hits().getAt(0).field("s_obj1").value(); + assertThat(sObj1.get("test").toString(), equalTo("something")); + assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something")); + + Map sObj2 = response.hits().getAt(0).field("s_obj2").value(); + List sObj2Arr2 = (List) sObj2.get("arr2"); + assertThat(sObj2Arr2.size(), equalTo(2)); + assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); + assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); + + sObj2Arr2 = (List) response.hits().getAt(0).field("s_obj2_arr2").value(); + assertThat(sObj2Arr2.size(), equalTo(2)); + assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); + assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); + + List sObj2Arr3 = (List) response.hits().getAt(0).field("s_arr3").value(); + assertThat(((Map) sObj2Arr3.get(0)).get("arr3_field1").toString(), equalTo("arr3_value1")); + } + + @Test + public void testPartialFields() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + client.admin().indices().prepareCreate("test").execute().actionGet(); + client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); + + + client.prepareIndex("test", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject() + .field("field1", "value1") + .startObject("obj1") + .startArray("arr1") + .startObject().startObject("obj2").field("field2", "value21").endObject().endObject() + .startObject().startObject("obj2").field("field2", "value22").endObject().endObject() + .endArray() + .endObject() + .endObject()) + .execute().actionGet(); + + client.admin().indices().prepareRefresh().execute().actionGet(); + + SearchResponse response = client.prepareSearch("test") + .addPartialField("partial1", "obj1.arr1.*", null) + .addPartialField("partial2", null, "obj1.*") + .execute().actionGet(); + assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0)); + + Map partial1 = response.hits().getAt(0).field("partial1").value(); + assertThat(partial1, notNullValue()); + assertThat(partial1.containsKey("field1"), equalTo(false)); + assertThat(partial1.containsKey("obj1"), equalTo(true)); + assertThat(((Map) partial1.get("obj1")).get("arr1"), instanceOf(List.class)); + + Map partial2 = response.hits().getAt(0).field("partial2").value(); + assertThat(partial2, notNullValue()); + assertThat(partial2.containsKey("obj1"), equalTo(false)); + assertThat(partial2.containsKey("field1"), equalTo(true)); + } } diff --git a/src/test/java/org/elasticsearch/test/integration/search/scriptfield/ScriptFieldSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/scriptfield/ScriptFieldSearchTests.java deleted file mode 100644 index 0e75bec50a8..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/search/scriptfield/ScriptFieldSearchTests.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.integration.search.scriptfield; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.collect.MapBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.test.integration.AbstractNodesTests; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.elasticsearch.client.Requests.refreshRequest; -import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; - -/** - * - */ -@Test -public class ScriptFieldSearchTests extends AbstractNodesTests { - - private Client client; - - @BeforeClass - public void createNodes() throws Exception { - startNode("server1", settingsBuilder().put("number_of_shards", 1).put("number_of_replicas", 0)); - startNode("client1", settingsBuilder().put("node.client", true).build()); - client = getClient(); - } - - @AfterClass - public void closeNodes() { - client.close(); - closeNode("client1"); - closeAllNodes(); - } - - protected Client getClient() { - return client("client1"); - } - - @Test - public void testDocAndFields() throws Exception { - try { - client.admin().indices().prepareDelete("test").execute().actionGet(); - } catch (Exception e) { - // its ok - } - client.admin().indices().prepareCreate("test").execute().actionGet(); - - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("num1").field("type", "double").field("store", "yes").endObject() - .endObject().endObject().endObject().string(); - - client.admin().indices().preparePutMapping().setType("type1").setSource(mapping).execute().actionGet(); - - client.prepareIndex("test", "type1", "1") - .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).field("date", "1970-01-01T00:00:00").endObject()) - .execute().actionGet(); - client.admin().indices().prepareFlush().execute().actionGet(); - client.prepareIndex("test", "type1", "2") - .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).field("date", "1970-01-01T00:00:25").endObject()) - .execute().actionGet(); - client.admin().indices().prepareFlush().execute().actionGet(); - client.prepareIndex("test", "type1", "3") - .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).field("date", "1970-01-01T00:02:00").endObject()) - .execute().actionGet(); - client.admin().indices().refresh(refreshRequest()).actionGet(); - - logger.info("running doc['num1'].value"); - SearchResponse response = client.prepareSearch() - .setQuery(matchAllQuery()) - .addSort("num1", SortOrder.ASC) - .addScriptField("sNum1", "doc['num1'].value") - .addScriptField("sNum1_field", "_fields['num1'].value") - .addScriptField("date1", "doc['date'].date.millis") - .execute().actionGet(); - - assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0)); - - assertThat(response.hits().totalHits(), equalTo(3l)); - assertThat(response.hits().getAt(0).isSourceEmpty(), equalTo(true)); - assertThat(response.hits().getAt(0).id(), equalTo("1")); - assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0)); - assertThat((Double) response.hits().getAt(0).fields().get("sNum1_field").values().get(0), equalTo(1.0)); - assertThat((Long) response.hits().getAt(0).fields().get("date1").values().get(0), equalTo(0l)); - assertThat(response.hits().getAt(1).id(), equalTo("2")); - assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0)); - assertThat((Double) response.hits().getAt(1).fields().get("sNum1_field").values().get(0), equalTo(2.0)); - assertThat((Long) response.hits().getAt(1).fields().get("date1").values().get(0), equalTo(25000l)); - assertThat(response.hits().getAt(2).id(), equalTo("3")); - assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0)); - assertThat((Double) response.hits().getAt(2).fields().get("sNum1_field").values().get(0), equalTo(3.0)); - assertThat((Long) response.hits().getAt(2).fields().get("date1").values().get(0), equalTo(120000l)); - - logger.info("running doc['num1'].value * factor"); - Map params = MapBuilder.newMapBuilder().put("factor", 2.0).map(); - response = client.prepareSearch() - .setQuery(matchAllQuery()) - .addSort("num1", SortOrder.ASC) - .addScriptField("sNum1", "doc['num1'].value * factor", params) - .execute().actionGet(); - - assertThat(response.hits().totalHits(), equalTo(3l)); - assertThat(response.hits().getAt(0).id(), equalTo("1")); - assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0)); - assertThat(response.hits().getAt(1).id(), equalTo("2")); - assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(4.0)); - assertThat(response.hits().getAt(2).id(), equalTo("3")); - assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(6.0)); - } - - @Test - public void testScriptFieldUsingSource() throws Exception { - try { - client.admin().indices().prepareDelete("test").execute().actionGet(); - } catch (Exception e) { - // its ok - } - client.admin().indices().prepareCreate("test").execute().actionGet(); - client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); - client.prepareIndex("test", "type1", "1") - .setSource(jsonBuilder().startObject() - .startObject("obj1").field("test", "something").endObject() - .startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject() - .startArray("arr3").startObject().field("arr3_field1", "arr3_value1").endObject().endArray() - .endObject()) - .execute().actionGet(); - client.admin().indices().refresh(refreshRequest()).actionGet(); - - SearchResponse response = client.prepareSearch() - .setQuery(matchAllQuery()) - .addField("_source.obj1") // we also automatically detect _source in fields - .addScriptField("s_obj1", "_source.obj1") - .addScriptField("s_obj1_test", "_source.obj1.test") - .addScriptField("s_obj2", "_source.obj2") - .addScriptField("s_obj2_arr2", "_source.obj2.arr2") - .addScriptField("s_arr3", "_source.arr3") - .execute().actionGet(); - - assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0)); - - Map sObj1 = (Map) response.hits().getAt(0).field("_source.obj1").value(); - assertThat(sObj1.get("test").toString(), equalTo("something")); - assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something")); - - sObj1 = (Map) response.hits().getAt(0).field("s_obj1").value(); - assertThat(sObj1.get("test").toString(), equalTo("something")); - assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something")); - - Map sObj2 = (Map) response.hits().getAt(0).field("s_obj2").value(); - List sObj2Arr2 = (List) sObj2.get("arr2"); - assertThat(sObj2Arr2.size(), equalTo(2)); - assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); - assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); - - sObj2Arr2 = (List) response.hits().getAt(0).field("s_obj2_arr2").value(); - assertThat(sObj2Arr2.size(), equalTo(2)); - assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); - assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); - - List sObj2Arr3 = (List) response.hits().getAt(0).field("s_arr3").value(); - assertThat(((Map) sObj2Arr3.get(0)).get("arr3_field1").toString(), equalTo("arr3_value1")); - } -} \ No newline at end of file diff --git a/src/test/java/org/elasticsearch/test/unit/common/xcontent/support/XContentMapValuesTests.java b/src/test/java/org/elasticsearch/test/unit/common/xcontent/support/XContentMapValuesTests.java index 48d65d6b6e2..83c73e13707 100644 --- a/src/test/java/org/elasticsearch/test/unit/common/xcontent/support/XContentMapValuesTests.java +++ b/src/test/java/org/elasticsearch/test/unit/common/xcontent/support/XContentMapValuesTests.java @@ -80,6 +80,10 @@ public class XContentMapValuesTests { filter = XContentMapValues.filter(source, new String[]{"test1*"}, Strings.EMPTY_ARRAY); assertThat(filter.get("test1"), equalTo(source.get("test1"))); assertThat(filter.containsKey("path1"), equalTo(false)); + + filter = XContentMapValues.filter(source, new String[]{"path1.path2.*"}, Strings.EMPTY_ARRAY); + assertThat(filter.get("path1"), equalTo(source.get("path1"))); + assertThat(filter.containsKey("test1"), equalTo(false)); } @SuppressWarnings({"unchecked"}) From 661d04e9de03995a9b410943afc2a661f24da9fb Mon Sep 17 00:00:00 2001 From: bbgordonn Date: Wed, 14 Dec 2011 17:40:21 -0500 Subject: [PATCH 077/270] #1452 closed: block writes or metadata changes if {index,cluster}.read_only is set. --- .../TransportClusterUpdateSettingsAction.java | 17 +- .../cluster/metadata/IndexMetaData.java | 15 ++ .../cluster/metadata/MetaData.java | 6 + .../MetaDataUpdateSettingsService.java | 16 +- .../elasticsearch/gateway/GatewayService.java | 8 + .../cluster/ClusterBlockTests.java | 202 ++++++++++++++++++ 6 files changed, 260 insertions(+), 4 deletions(-) create mode 100644 src/test/java/org/elasticsearch/test/integration/cluster/ClusterBlockTests.java diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java index 7531a1b22a5..29a88c7d416 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.ProcessedClusterStateUpdateTask; +import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; @@ -111,12 +112,22 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeOpe return currentState; } + Settings persistentSettingsBuilt = persistentSettings.build(); + Settings transientSettingsBuilt = transientSettings.build(); MetaData.Builder metaData = MetaData.builder().metaData(currentState.metaData()) - .persistentSettings(persistentSettings.build()) - .transientSettings(transientSettings.build()); + .persistentSettings(persistentSettingsBuilt) + .transientSettings(transientSettingsBuilt); + ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks()); + Boolean updatedReadOnly = persistentSettingsBuilt.getAsBoolean(MetaData.SETTING_READ_ONLY, false) || transientSettingsBuilt.getAsBoolean(MetaData.SETTING_READ_ONLY, false); + if (updatedReadOnly) { + blocks.addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK); + } + else { + blocks.removeGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK); + } - return ClusterState.builder().state(currentState).metaData(metaData).build(); + return ClusterState.builder().state(currentState).metaData(metaData).blocks(blocks).build(); } catch (Exception e) { latch.countDown(); logger.warn("failed to update cluster settings", e); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 98486fd9d39..60ff5618160 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -22,6 +22,8 @@ package org.elasticsearch.cluster.metadata; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.elasticsearch.ElasticSearchIllegalStateException; +import org.elasticsearch.cluster.block.ClusterBlock; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.node.DiscoveryNodeFilters; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Preconditions; @@ -51,8 +53,11 @@ public class IndexMetaData { private static ImmutableSet dynamicSettings = ImmutableSet.builder() .add(IndexMetaData.SETTING_NUMBER_OF_REPLICAS) .add(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS) + .add(IndexMetaData.SETTING_READ_ONLY) .build(); + public static final ClusterBlock INDEX_READ_ONLY_BLOCK = new ClusterBlock(5, "index read-only", false, false, ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA); + public static ImmutableSet dynamicSettings() { return dynamicSettings; } @@ -111,6 +116,8 @@ public class IndexMetaData { public static final String SETTING_AUTO_EXPAND_REPLICAS = "index.auto_expand_replicas"; + public static final String SETTING_READ_ONLY = "index.read_only"; + private final String index; private final State state; @@ -191,6 +198,14 @@ public class IndexMetaData { return totalNumberOfShards(); } + public boolean readOnly() { + return settings.getAsBoolean(SETTING_READ_ONLY, false); + } + + public boolean getreadOnly() { + return readOnly(); + } + public Settings settings() { return settings; } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 793cb22fa1b..e4fdf844196 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -22,6 +22,8 @@ package org.elasticsearch.cluster.metadata; import com.google.common.collect.*; import gnu.trove.set.hash.THashSet; import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.cluster.block.ClusterBlock; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.MapBuilder; @@ -47,8 +49,12 @@ import static org.elasticsearch.common.settings.ImmutableSettings.*; * */ public class MetaData implements Iterable { + public static final String SETTING_READ_ONLY = "cluster.read_only"; + + public static final ClusterBlock CLUSTER_READ_ONLY_BLOCK = new ClusterBlock(6, "cluster read-only", false, false, ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA); private static ImmutableSet dynamicSettings = ImmutableSet.builder() + .add("cluster.read_only") .build(); public static ImmutableSet dynamicSettings() { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java index 50b039f23a7..feffb29c7c5 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java @@ -22,6 +22,7 @@ package org.elasticsearch.cluster.metadata; import com.google.common.collect.Sets; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.cluster.*; +import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; @@ -163,6 +164,19 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements logger.info("updating number_of_replicas to [{}] for indices {}", updatedNumberOfReplicas, actualIndices); } + ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks()); + Boolean updatedReadOnly = openSettings.getAsBoolean(IndexMetaData.SETTING_READ_ONLY, null); + if (updatedReadOnly != null) { + for (String index : actualIndices) { + if (updatedReadOnly) { + blocks.addIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_BLOCK); + } + else { + blocks.removeIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_BLOCK); + } + } + } + // allow to change any settings to a close index, and only allow dynamic settings to be changed // on an open index Set openIndices = Sets.newHashSet(); @@ -189,7 +203,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements } - ClusterState updatedState = ClusterState.builder().state(currentState).metaData(metaDataBuilder).routingTable(routingTableBuilder).build(); + ClusterState updatedState = ClusterState.builder().state(currentState).metaData(metaDataBuilder).routingTable(routingTableBuilder).blocks(blocks).build(); // now, reroute in case things change that require it (like number of replicas) RoutingAllocation.Result routingResult = allocationService.reroute(updatedState); diff --git a/src/main/java/org/elasticsearch/gateway/GatewayService.java b/src/main/java/org/elasticsearch/gateway/GatewayService.java index 13e1a4722ca..508d8b64008 100644 --- a/src/main/java/org/elasticsearch/gateway/GatewayService.java +++ b/src/main/java/org/elasticsearch/gateway/GatewayService.java @@ -258,11 +258,19 @@ public class GatewayService extends AbstractLifecycleComponent i metaDataBuilder.put(entry.getValue()); } + + if (recoveredState.metaData().settings().getAsBoolean(MetaData.SETTING_READ_ONLY, false) || currentState.metaData().settings().getAsBoolean(MetaData.SETTING_READ_ONLY, false)) { + blocks.addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK); + } + for (IndexMetaData indexMetaData : recoveredState.metaData()) { metaDataBuilder.put(indexMetaData); if (indexMetaData.state() == IndexMetaData.State.CLOSE) { blocks.addIndexBlock(indexMetaData.index(), MetaDataStateIndexService.INDEX_CLOSED_BLOCK); } + if (indexMetaData.readOnly()) { + blocks.addIndexBlock(indexMetaData.index(), IndexMetaData.INDEX_READ_ONLY_BLOCK); + } } // update the state to reflect the new metadata and routing diff --git a/src/test/java/org/elasticsearch/test/integration/cluster/ClusterBlockTests.java b/src/test/java/org/elasticsearch/test/integration/cluster/ClusterBlockTests.java new file mode 100644 index 00000000000..94f17693c0d --- /dev/null +++ b/src/test/java/org/elasticsearch/test/integration/cluster/ClusterBlockTests.java @@ -0,0 +1,202 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.metadata; + +import java.util.HashMap; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.exists.IndicesExistsResponse; +import org.elasticsearch.action.admin.indices.settings.UpdateSettingsResponse; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.action.admin.cluster.settings.ClusterUpdateSettingsRequestBuilder; +import org.elasticsearch.client.action.admin.indices.settings.UpdateSettingsRequestBuilder; +import org.elasticsearch.client.action.index.IndexRequestBuilder; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeBuilder; +import org.testng.annotations.Test; + +import static org.elasticsearch.node.NodeBuilder.*; +import static org.hamcrest.MatcherAssert.*; +import static org.hamcrest.Matchers.*; + +@Test +public class ClusterBlockTests { + @Test public void testClusterReadOnly() throws Exception { + Node node = newNode(); + try { + Client client = node.client(); + try { + // cluster.read_only = null: write and metadata not blocked + canCreateIndex(client, "test1"); + canIndexDocument(client, "test1"); + setIndexReadOnly(client, "test1", "false"); + canIndexExists(client, "test1"); + + // cluster.read_only = true: block write and metadata + setClusterReadOnly(client, "true"); + canNotCreateIndex(client, "test2"); + // even if index has index.read_only = false + canNotIndexDocument(client, "test1"); + canNotIndexExists(client, "test1"); + + // cluster.read_only = false: removes the block + setClusterReadOnly(client, "false"); + canCreateIndex(client, "test2"); + canIndexDocument(client, "test2"); + canIndexDocument(client, "test1"); + canIndexExists(client, "test1"); + } + finally { + client.close(); + } + } + finally { + node.close(); + } + } + + @Test public void testIndexReadOnly() throws Exception { + Node node = newNode(); + try { + Client client = node.client(); + try { + // newly created an index has no blocks + canCreateIndex(client, "ro"); + canIndexDocument(client, "ro"); + canIndexExists(client, "ro"); + + // adds index write and metadata block + setIndexReadOnly(client, "ro", "true"); + canNotIndexDocument(client, "ro"); + canNotIndexExists(client, "ro"); + + // other indices not blocked + canCreateIndex(client, "rw"); + canIndexDocument(client, "rw"); + canIndexExists(client, "rw"); + + // blocks can be removed + setIndexReadOnly(client, "ro", "false"); + canIndexDocument(client, "ro"); + canIndexExists(client, "ro"); + } + finally { + client.close(); + } + } + finally { + node.close(); + } + } + + private Node newNode() { + ImmutableSettings.Builder settingsBuilder = ImmutableSettings.settingsBuilder().put("gateway.type", "none"); + NodeBuilder nodeBuilder = nodeBuilder().local(true).loadConfigSettings(false).clusterName("ClusterBlockTests").settings(settingsBuilder); + return nodeBuilder.node(); + } + + private void canCreateIndex(Client client, String index) { + try { + CreateIndexResponse r = client.admin().indices().prepareCreate(index).execute().actionGet(); + assertThat(r, notNullValue()); + } + catch (ClusterBlockException e) { + assert false; + } + } + + private void canNotCreateIndex(Client client, String index) { + try { + client.admin().indices().prepareCreate(index).execute().actionGet(); + assert false; + } + catch (ClusterBlockException e) { + // all is well + } + } + + private void canIndexDocument(Client client, String index) { + try { + IndexRequestBuilder builder = client.prepareIndex(index, "zzz"); + builder.setSource("foo", "bar"); + IndexResponse r = builder.execute().actionGet(); + assertThat(r, notNullValue()); + } + catch (ClusterBlockException e) { + assert false; + } + } + + private void canNotIndexDocument(Client client, String index) { + try { + IndexRequestBuilder builder = client.prepareIndex(index, "zzz"); + builder.setSource("foo", "bar"); + builder.execute().actionGet(); + assert false; + } + catch (ClusterBlockException e) { + // all is well + } + } + + private void canIndexExists(Client client, String index) { + try { + IndicesExistsResponse r = client.admin().indices().prepareExists(index).execute().actionGet(); + assertThat(r, notNullValue()); + } + catch (ClusterBlockException e) { + assert false; + } + } + + private void canNotIndexExists(Client client, String index) { + try { + IndicesExistsResponse r = client.admin().indices().prepareExists(index).execute().actionGet(); + assert false; + } + catch (ClusterBlockException e) { + // all is well + } + } + + private void setClusterReadOnly(Client client, String value) { + HashMap newSettings = new HashMap(); + newSettings.put(MetaData.SETTING_READ_ONLY, value); + + ClusterUpdateSettingsRequestBuilder settingsRequest = client.admin().cluster().prepareUpdateSettings(); + settingsRequest.setTransientSettings(newSettings); + ClusterUpdateSettingsResponse settingsResponse = settingsRequest.execute().actionGet(); + assertThat(settingsResponse, notNullValue()); + } + + private void setIndexReadOnly(Client client, String index, Object value) { + HashMap newSettings = new HashMap(); + newSettings.put(IndexMetaData.SETTING_READ_ONLY, value); + + UpdateSettingsRequestBuilder settingsRequest = client.admin().indices().prepareUpdateSettings(index); + settingsRequest.setSettings(newSettings); + UpdateSettingsResponse settingsResponse = settingsRequest.execute().actionGet(); + assertThat(settingsResponse, notNullValue()); + } +} From b41166a78ad684e29110ebec948f0263b1fca9ed Mon Sep 17 00:00:00 2001 From: bbgordonn Date: Wed, 14 Dec 2011 18:20:03 -0500 Subject: [PATCH 078/270] RestIndicesExistsAction: include block message in error if ClusterBlockException. --- .../action/admin/indices/exists/RestIndicesExistsAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/RestIndicesExistsAction.java b/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/RestIndicesExistsAction.java index b2a34e0c372..f8cc00640cc 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/RestIndicesExistsAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/RestIndicesExistsAction.java @@ -70,11 +70,11 @@ public class RestIndicesExistsAction extends BaseRestHandler { @Override public void onFailure(Throwable e) { try { - channel.sendResponse(new StringRestResponse(INTERNAL_SERVER_ERROR)); + channel.sendResponse(new XContentThrowableRestResponse(request, e)); } catch (Exception e1) { logger.error("Failed to send failure response", e1); } } }); } -} \ No newline at end of file +} From 5049f60b6ccb0203ca01c828082ec153d662e9d3 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 27 Dec 2011 20:35:07 +0200 Subject: [PATCH 079/270] Set an index / indices to read only, or make the cluster read only, closes #1573. --- .../TransportClusterUpdateSettingsAction.java | 11 +- .../cluster/metadata/IndexMetaData.java | 15 +- .../cluster/metadata/MetaData.java | 12 +- .../MetaDataUpdateSettingsService.java | 3 +- .../elasticsearch/gateway/GatewayService.java | 2 +- .../ClusterAndIndexReaderOnlyTests.java} | 144 +++++++----------- 6 files changed, 77 insertions(+), 110 deletions(-) rename src/test/java/org/elasticsearch/test/integration/{cluster/ClusterBlockTests.java => readonly/ClusterAndIndexReaderOnlyTests.java} (58%) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java index 29a88c7d416..85482257de9 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java @@ -112,18 +112,15 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeOpe return currentState; } - Settings persistentSettingsBuilt = persistentSettings.build(); - Settings transientSettingsBuilt = transientSettings.build(); MetaData.Builder metaData = MetaData.builder().metaData(currentState.metaData()) - .persistentSettings(persistentSettingsBuilt) - .transientSettings(transientSettingsBuilt); + .persistentSettings(persistentSettings.build()) + .transientSettings(transientSettings.build()); ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks()); - Boolean updatedReadOnly = persistentSettingsBuilt.getAsBoolean(MetaData.SETTING_READ_ONLY, false) || transientSettingsBuilt.getAsBoolean(MetaData.SETTING_READ_ONLY, false); + boolean updatedReadOnly = metaData.persistentSettings().getAsBoolean(MetaData.SETTING_READ_ONLY, false) || metaData.transientSettings().getAsBoolean(MetaData.SETTING_READ_ONLY, false); if (updatedReadOnly) { blocks.addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK); - } - else { + } else { blocks.removeGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK); } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 60ff5618160..cf01f742fd6 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -56,7 +56,7 @@ public class IndexMetaData { .add(IndexMetaData.SETTING_READ_ONLY) .build(); - public static final ClusterBlock INDEX_READ_ONLY_BLOCK = new ClusterBlock(5, "index read-only", false, false, ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA); + public static final ClusterBlock INDEX_READ_ONLY_BLOCK = new ClusterBlock(5, "index read-only (api)", false, false, ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA); public static ImmutableSet dynamicSettings() { return dynamicSettings; @@ -111,12 +111,9 @@ public class IndexMetaData { } public static final String SETTING_NUMBER_OF_SHARDS = "index.number_of_shards"; - public static final String SETTING_NUMBER_OF_REPLICAS = "index.number_of_replicas"; - public static final String SETTING_AUTO_EXPAND_REPLICAS = "index.auto_expand_replicas"; - - public static final String SETTING_READ_ONLY = "index.read_only"; + public static final String SETTING_READ_ONLY = "index.blocks.read_only"; private final String index; @@ -198,14 +195,6 @@ public class IndexMetaData { return totalNumberOfShards(); } - public boolean readOnly() { - return settings.getAsBoolean(SETTING_READ_ONLY, false); - } - - public boolean getreadOnly() { - return readOnly(); - } - public Settings settings() { return settings; } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index e4fdf844196..916637fd61b 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -49,9 +49,9 @@ import static org.elasticsearch.common.settings.ImmutableSettings.*; * */ public class MetaData implements Iterable { - public static final String SETTING_READ_ONLY = "cluster.read_only"; + public static final String SETTING_READ_ONLY = "cluster.blocks.read_only"; - public static final ClusterBlock CLUSTER_READ_ONLY_BLOCK = new ClusterBlock(6, "cluster read-only", false, false, ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA); + public static final ClusterBlock CLUSTER_READ_ONLY_BLOCK = new ClusterBlock(6, "cluster read-only (api)", false, false, ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA); private static ImmutableSet dynamicSettings = ImmutableSet.builder() .add("cluster.read_only") @@ -718,11 +718,19 @@ public class MetaData implements Iterable { return this; } + public Settings transientSettings() { + return this.transientSettings; + } + public Builder transientSettings(Settings settings) { this.transientSettings = settings; return this; } + public Settings persistentSettings() { + return this.persistentSettings; + } + public Builder persistentSettings(Settings settings) { this.persistentSettings = settings; return this; diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java index feffb29c7c5..e486310354c 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java @@ -170,8 +170,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements for (String index : actualIndices) { if (updatedReadOnly) { blocks.addIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_BLOCK); - } - else { + } else { blocks.removeIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_BLOCK); } } diff --git a/src/main/java/org/elasticsearch/gateway/GatewayService.java b/src/main/java/org/elasticsearch/gateway/GatewayService.java index 508d8b64008..75b7f9e1aa8 100644 --- a/src/main/java/org/elasticsearch/gateway/GatewayService.java +++ b/src/main/java/org/elasticsearch/gateway/GatewayService.java @@ -268,7 +268,7 @@ public class GatewayService extends AbstractLifecycleComponent i if (indexMetaData.state() == IndexMetaData.State.CLOSE) { blocks.addIndexBlock(indexMetaData.index(), MetaDataStateIndexService.INDEX_CLOSED_BLOCK); } - if (indexMetaData.readOnly()) { + if (indexMetaData.settings().getAsBoolean(IndexMetaData.SETTING_READ_ONLY, false)) { blocks.addIndexBlock(indexMetaData.index(), IndexMetaData.INDEX_READ_ONLY_BLOCK); } } diff --git a/src/test/java/org/elasticsearch/test/integration/cluster/ClusterBlockTests.java b/src/test/java/org/elasticsearch/test/integration/readonly/ClusterAndIndexReaderOnlyTests.java similarity index 58% rename from src/test/java/org/elasticsearch/test/integration/cluster/ClusterBlockTests.java rename to src/test/java/org/elasticsearch/test/integration/readonly/ClusterAndIndexReaderOnlyTests.java index 94f17693c0d..b3a61be9185 100644 --- a/src/test/java/org/elasticsearch/test/integration/cluster/ClusterBlockTests.java +++ b/src/test/java/org/elasticsearch/test/integration/readonly/ClusterAndIndexReaderOnlyTests.java @@ -17,9 +17,8 @@ * under the License. */ -package org.elasticsearch.cluster.metadata; +package org.elasticsearch.test.integration.readonly; -import java.util.HashMap; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.exists.IndicesExistsResponse; @@ -31,97 +30,77 @@ import org.elasticsearch.client.action.admin.indices.settings.UpdateSettingsRequ import org.elasticsearch.client.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; +import org.elasticsearch.test.integration.AbstractNodesTests; +import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; -import static org.elasticsearch.node.NodeBuilder.*; -import static org.hamcrest.MatcherAssert.*; -import static org.hamcrest.Matchers.*; +import java.util.HashMap; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.notNullValue; @Test -public class ClusterBlockTests { - @Test public void testClusterReadOnly() throws Exception { - Node node = newNode(); - try { - Client client = node.client(); - try { - // cluster.read_only = null: write and metadata not blocked - canCreateIndex(client, "test1"); - canIndexDocument(client, "test1"); - setIndexReadOnly(client, "test1", "false"); - canIndexExists(client, "test1"); +public class ClusterAndIndexReaderOnlyTests extends AbstractNodesTests { - // cluster.read_only = true: block write and metadata - setClusterReadOnly(client, "true"); - canNotCreateIndex(client, "test2"); - // even if index has index.read_only = false - canNotIndexDocument(client, "test1"); - canNotIndexExists(client, "test1"); - - // cluster.read_only = false: removes the block - setClusterReadOnly(client, "false"); - canCreateIndex(client, "test2"); - canIndexDocument(client, "test2"); - canIndexDocument(client, "test1"); - canIndexExists(client, "test1"); - } - finally { - client.close(); - } - } - finally { - node.close(); - } + @AfterMethod + public void closeNodes() { + closeAllNodes(); } - @Test public void testIndexReadOnly() throws Exception { - Node node = newNode(); - try { - Client client = node.client(); - try { - // newly created an index has no blocks - canCreateIndex(client, "ro"); - canIndexDocument(client, "ro"); - canIndexExists(client, "ro"); + @Test + public void verifyReadOnly() throws Exception { + Node node1 = startNode("node1"); + Client client = node1.client(); - // adds index write and metadata block - setIndexReadOnly(client, "ro", "true"); - canNotIndexDocument(client, "ro"); - canNotIndexExists(client, "ro"); + // cluster.read_only = null: write and metadata not blocked + canCreateIndex(client, "test1"); + canIndexDocument(client, "test1"); + setIndexReadOnly(client, "test1", "false"); + canIndexExists(client, "test1"); - // other indices not blocked - canCreateIndex(client, "rw"); - canIndexDocument(client, "rw"); - canIndexExists(client, "rw"); + // cluster.read_only = true: block write and metadata + setClusterReadOnly(client, "true"); + canNotCreateIndex(client, "test2"); + // even if index has index.read_only = false + canNotIndexDocument(client, "test1"); + canNotIndexExists(client, "test1"); - // blocks can be removed - setIndexReadOnly(client, "ro", "false"); - canIndexDocument(client, "ro"); - canIndexExists(client, "ro"); - } - finally { - client.close(); - } - } - finally { - node.close(); - } - } + // cluster.read_only = false: removes the block + setClusterReadOnly(client, "false"); + canCreateIndex(client, "test2"); + canIndexDocument(client, "test2"); + canIndexDocument(client, "test1"); + canIndexExists(client, "test1"); - private Node newNode() { - ImmutableSettings.Builder settingsBuilder = ImmutableSettings.settingsBuilder().put("gateway.type", "none"); - NodeBuilder nodeBuilder = nodeBuilder().local(true).loadConfigSettings(false).clusterName("ClusterBlockTests").settings(settingsBuilder); - return nodeBuilder.node(); + + // newly created an index has no blocks + canCreateIndex(client, "ro"); + canIndexDocument(client, "ro"); + canIndexExists(client, "ro"); + + // adds index write and metadata block + setIndexReadOnly(client, "ro", "true"); + canNotIndexDocument(client, "ro"); + canNotIndexExists(client, "ro"); + + // other indices not blocked + canCreateIndex(client, "rw"); + canIndexDocument(client, "rw"); + canIndexExists(client, "rw"); + + // blocks can be removed + setIndexReadOnly(client, "ro", "false"); + canIndexDocument(client, "ro"); + canIndexExists(client, "ro"); } private void canCreateIndex(Client client, String index) { try { CreateIndexResponse r = client.admin().indices().prepareCreate(index).execute().actionGet(); assertThat(r, notNullValue()); - } - catch (ClusterBlockException e) { + } catch (ClusterBlockException e) { assert false; } } @@ -130,8 +109,7 @@ public class ClusterBlockTests { try { client.admin().indices().prepareCreate(index).execute().actionGet(); assert false; - } - catch (ClusterBlockException e) { + } catch (ClusterBlockException e) { // all is well } } @@ -142,8 +120,7 @@ public class ClusterBlockTests { builder.setSource("foo", "bar"); IndexResponse r = builder.execute().actionGet(); assertThat(r, notNullValue()); - } - catch (ClusterBlockException e) { + } catch (ClusterBlockException e) { assert false; } } @@ -154,8 +131,7 @@ public class ClusterBlockTests { builder.setSource("foo", "bar"); builder.execute().actionGet(); assert false; - } - catch (ClusterBlockException e) { + } catch (ClusterBlockException e) { // all is well } } @@ -164,8 +140,7 @@ public class ClusterBlockTests { try { IndicesExistsResponse r = client.admin().indices().prepareExists(index).execute().actionGet(); assertThat(r, notNullValue()); - } - catch (ClusterBlockException e) { + } catch (ClusterBlockException e) { assert false; } } @@ -174,8 +149,7 @@ public class ClusterBlockTests { try { IndicesExistsResponse r = client.admin().indices().prepareExists(index).execute().actionGet(); assert false; - } - catch (ClusterBlockException e) { + } catch (ClusterBlockException e) { // all is well } } From f79cf4149fe680ff36117277ffb45d4f2e53c33d Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 27 Dec 2011 20:36:57 +0200 Subject: [PATCH 080/270] remove unused code --- .../java/org/elasticsearch/gateway/GatewayService.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/gateway/GatewayService.java b/src/main/java/org/elasticsearch/gateway/GatewayService.java index 75b7f9e1aa8..47683331930 100644 --- a/src/main/java/org/elasticsearch/gateway/GatewayService.java +++ b/src/main/java/org/elasticsearch/gateway/GatewayService.java @@ -24,7 +24,10 @@ import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.block.ClusterBlocks; -import org.elasticsearch.cluster.metadata.*; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.MetaDataStateIndexService; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.RoutingTable; @@ -62,8 +65,6 @@ public class GatewayService extends AbstractLifecycleComponent i private final DiscoveryService discoveryService; - private final MetaDataCreateIndexService createIndexService; - private final TimeValue recoverAfterTime; private final int recoverAfterNodes; private final int expectedNodes; @@ -77,13 +78,12 @@ public class GatewayService extends AbstractLifecycleComponent i private final AtomicBoolean scheduledRecovery = new AtomicBoolean(); @Inject - public GatewayService(Settings settings, Gateway gateway, AllocationService allocationService, ClusterService clusterService, DiscoveryService discoveryService, MetaDataCreateIndexService createIndexService, ThreadPool threadPool) { + public GatewayService(Settings settings, Gateway gateway, AllocationService allocationService, ClusterService clusterService, DiscoveryService discoveryService, ThreadPool threadPool) { super(settings); this.gateway = gateway; this.allocationService = allocationService; this.clusterService = clusterService; this.discoveryService = discoveryService; - this.createIndexService = createIndexService; this.threadPool = threadPool; // allow to control a delay of when indices will get created this.recoverAfterTime = componentSettings.getAsTime("recover_after_time", null); From be6e18cb3652da731d9877af48d77fd261207334 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 22 Dec 2011 16:16:18 -0700 Subject: [PATCH 081/270] Add query validation feature --- .../action/TransportActions.java | 2 + .../action/validate/ShardValidateRequest.java | 120 +++++++++ .../validate/ShardValidateResponse.java | 61 +++++ .../validate/TransportValidateAction.java | 141 ++++++++++ .../action/validate/ValidateRequest.java | 254 ++++++++++++++++++ .../action/validate/ValidateResponse.java | 73 +++++ .../action/validate/pacakge-info.java | 23 ++ .../java/org/elasticsearch/client/Client.java | 26 ++ .../validate/ValidateRequestBuilder.java | 75 ++++++ .../elasticsearch/client/node/NodeClient.java | 19 +- .../client/support/AbstractClient.java | 6 + .../client/transport/TransportClient.java | 12 + .../ClientTransportValidateAction.java | 25 ++ .../support/InternalTransportClient.java | 28 +- .../index/shard/service/IndexShard.java | 4 + .../shard/service/InternalIndexShard.java | 23 ++ .../rest/action/RestActionModule.java | 3 + .../action/validate/RestValidateAction.java | 122 +++++++++ .../validate/SimpleValidateTests.java | 84 ++++++ 19 files changed, 1099 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/elasticsearch/action/validate/ShardValidateRequest.java create mode 100644 src/main/java/org/elasticsearch/action/validate/ShardValidateResponse.java create mode 100644 src/main/java/org/elasticsearch/action/validate/TransportValidateAction.java create mode 100644 src/main/java/org/elasticsearch/action/validate/ValidateRequest.java create mode 100644 src/main/java/org/elasticsearch/action/validate/ValidateResponse.java create mode 100644 src/main/java/org/elasticsearch/action/validate/pacakge-info.java create mode 100644 src/main/java/org/elasticsearch/client/action/validate/ValidateRequestBuilder.java create mode 100644 src/main/java/org/elasticsearch/client/transport/action/validate/ClientTransportValidateAction.java create mode 100644 src/main/java/org/elasticsearch/rest/action/validate/RestValidateAction.java create mode 100644 src/test/java/org/elasticsearch/test/integration/validate/SimpleValidateTests.java diff --git a/src/main/java/org/elasticsearch/action/TransportActions.java b/src/main/java/org/elasticsearch/action/TransportActions.java index a10a445d76b..f3826e17405 100644 --- a/src/main/java/org/elasticsearch/action/TransportActions.java +++ b/src/main/java/org/elasticsearch/action/TransportActions.java @@ -29,6 +29,8 @@ public class TransportActions { public static final String INDEX = "indices/index/shard/index"; public static final String COUNT = "indices/count"; + + public static final String VALIDATE = "indices/validate"; public static final String DELETE = "indices/index/shard/delete"; diff --git a/src/main/java/org/elasticsearch/action/validate/ShardValidateRequest.java b/src/main/java/org/elasticsearch/action/validate/ShardValidateRequest.java new file mode 100644 index 00000000000..d275a43c706 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/validate/ShardValidateRequest.java @@ -0,0 +1,120 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.validate; + +import org.elasticsearch.action.support.broadcast.BroadcastShardOperationRequest; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +/** + * Internal validate request executed directly against a specific index shard. + * + * + */ +class ShardValidateRequest extends BroadcastShardOperationRequest { + + private byte[] querySource; + private int querySourceOffset; + private int querySourceLength; + + private String[] types = Strings.EMPTY_ARRAY; + + @Nullable + private String[] filteringAliases; + + ShardValidateRequest() { + + } + + public ShardValidateRequest(String index, int shardId, @Nullable String[] filteringAliases, ValidateRequest request) { + super(index, shardId); + this.querySource = request.querySource(); + this.querySourceOffset = request.querySourceOffset(); + this.querySourceLength = request.querySourceLength(); + this.types = request.types(); + this.filteringAliases = filteringAliases; + } + + public byte[] querySource() { + return querySource; + } + + public int querySourceOffset() { + return querySourceOffset; + } + + public int querySourceLength() { + return querySourceLength; + } + + public String[] types() { + return this.types; + } + + public String[] filteringAliases() { + return filteringAliases; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + querySourceLength = in.readVInt(); + querySourceOffset = 0; + querySource = new byte[querySourceLength]; + in.readFully(querySource); + int typesSize = in.readVInt(); + if (typesSize > 0) { + types = new String[typesSize]; + for (int i = 0; i < typesSize; i++) { + types[i] = in.readUTF(); + } + } + int aliasesSize = in.readVInt(); + if (aliasesSize > 0) { + filteringAliases = new String[aliasesSize]; + for (int i = 0; i < aliasesSize; i++) { + filteringAliases[i] = in.readUTF(); + } + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeVInt(querySourceLength); + out.writeBytes(querySource, querySourceOffset, querySourceLength); + out.writeVInt(types.length); + for (String type : types) { + out.writeUTF(type); + } + if (filteringAliases != null) { + out.writeVInt(filteringAliases.length); + for (String alias : filteringAliases) { + out.writeUTF(alias); + } + } else { + out.writeVInt(0); + } + } +} diff --git a/src/main/java/org/elasticsearch/action/validate/ShardValidateResponse.java b/src/main/java/org/elasticsearch/action/validate/ShardValidateResponse.java new file mode 100644 index 00000000000..31b16cddcc7 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/validate/ShardValidateResponse.java @@ -0,0 +1,61 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.validate; + +import org.elasticsearch.action.support.broadcast.BroadcastShardOperationResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +/** + * Internal validate response of a shard validate request executed directly against a specific shard. + * + * + */ +class ShardValidateResponse extends BroadcastShardOperationResponse { + + private boolean valid; + + ShardValidateResponse() { + + } + + public ShardValidateResponse(String index, int shardId, boolean valid) { + super(index, shardId); + this.valid = valid; + } + + boolean valid() { + return this.valid; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + valid = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBoolean(valid); + } +} diff --git a/src/main/java/org/elasticsearch/action/validate/TransportValidateAction.java b/src/main/java/org/elasticsearch/action/validate/TransportValidateAction.java new file mode 100644 index 00000000000..d17771847ff --- /dev/null +++ b/src/main/java/org/elasticsearch/action/validate/TransportValidateAction.java @@ -0,0 +1,141 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.validate; + +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.action.ShardOperationFailedException; +import org.elasticsearch.action.TransportActions; +import org.elasticsearch.action.support.DefaultShardOperationFailedException; +import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; +import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; +import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.routing.GroupShardsIterator; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.shard.service.IndexShard; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReferenceArray; + +import static com.google.common.collect.Lists.newArrayList; + +/** + * + */ +public class TransportValidateAction extends TransportBroadcastOperationAction { + + private final IndicesService indicesService; + + @Inject + public TransportValidateAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, IndicesService indicesService) { + super(settings, threadPool, clusterService, transportService); + this.indicesService = indicesService; + } + + @Override + protected String executor() { + return ThreadPool.Names.SEARCH; + } + + @Override + protected String transportAction() { + return TransportActions.VALIDATE; + } + + @Override + protected String transportShardAction() { + return "indices/validate/shard"; + } + + @Override + protected ValidateRequest newRequest() { + return new ValidateRequest(); + } + + @Override + protected ShardValidateRequest newShardRequest() { + return new ShardValidateRequest(); + } + + @Override + protected ShardValidateRequest newShardRequest(ShardRouting shard, ValidateRequest request) { + String[] filteringAliases = clusterService.state().metaData().filteringAliases(shard.index(), request.indices()); + return new ShardValidateRequest(shard.index(), shard.id(), filteringAliases, request); + } + + @Override + protected ShardValidateResponse newShardResponse() { + return new ShardValidateResponse(); + } + + @Override + protected GroupShardsIterator shards(ValidateRequest request, String[] concreteIndices, ClusterState clusterState) { + // Hard-code routing to limit request to a single shard. + Map> routingMap = clusterState.metaData().resolveSearchRouting("0", request.indices()); + return clusterService.operationRouting().searchShards(clusterState, request.indices(), concreteIndices, null, routingMap, null); + } + + @Override + protected void checkBlock(ValidateRequest request, String[] concreteIndices, ClusterState state) { + for (String index : concreteIndices) { + state.blocks().indexBlocked(ClusterBlockLevel.READ, index); + } + } + + @Override + protected ValidateResponse newResponse(ValidateRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) { + int successfulShards = 0; + int failedShards = 0; + boolean valid = true; + List shardFailures = null; + for (int i = 0; i < shardsResponses.length(); i++) { + Object shardResponse = shardsResponses.get(i); + if (shardResponse == null) { + failedShards++; + } else if (shardResponse instanceof BroadcastShardOperationFailedException) { + failedShards++; + if (shardFailures == null) { + shardFailures = newArrayList(); + } + shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse)); + } else { + valid = valid && ((ShardValidateResponse) shardResponse).valid(); + successfulShards++; + } + } + return new ValidateResponse(valid, shardsResponses.length(), successfulShards, failedShards, shardFailures); + } + + @Override + protected ShardValidateResponse shardOperation(ShardValidateRequest request) throws ElasticSearchException { + IndexShard indexShard = indicesService.indexServiceSafe(request.index()).shardSafe(request.shardId()); + boolean valid = indexShard.validate(request.querySource(), request.querySourceOffset(), request.querySourceLength(), + request.filteringAliases(), request.types()); + return new ShardValidateResponse(request.index(), request.shardId(), valid); + } +} diff --git a/src/main/java/org/elasticsearch/action/validate/ValidateRequest.java b/src/main/java/org/elasticsearch/action/validate/ValidateRequest.java new file mode 100644 index 00000000000..d0f1505c404 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/validate/ValidateRequest.java @@ -0,0 +1,254 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.validate; + +import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.ElasticSearchGenerationException; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.broadcast.BroadcastOperationRequest; +import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; +import org.elasticsearch.client.Requests; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Required; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.Unicode; +import org.elasticsearch.common.io.BytesStream; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.QueryBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; + +/** + * A request to validate a specific query. + *

+ *

The request requires the query source to be set either using {@link #query(org.elasticsearch.index.query.QueryBuilder)}, + * or {@link #query(byte[])}. + * + * + */ +public class ValidateRequest extends BroadcastOperationRequest { + + private static final XContentType contentType = Requests.CONTENT_TYPE; + + private byte[] querySource; + private int querySourceOffset; + private int querySourceLength; + private boolean querySourceUnsafe; + + private String[] types = Strings.EMPTY_ARRAY; + + ValidateRequest() { + } + + /** + * Constructs a new validate request against the provided indices. No indices provided means it will + * run against all indices. + */ + public ValidateRequest(String... indices) { + super(indices); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = super.validate(); + return validationException; + } + + /** + * Controls the operation threading model. + */ + @Override + public ValidateRequest operationThreading(BroadcastOperationThreading operationThreading) { + super.operationThreading(operationThreading); + return this; + } + + @Override + protected void beforeStart() { + if (querySourceUnsafe) { + querySource = Arrays.copyOfRange(querySource, querySourceOffset, querySourceOffset + querySourceLength); + querySourceOffset = 0; + querySourceUnsafe = false; + } + } + + /** + * Should the listener be called on a separate thread if needed. + */ + @Override + public ValidateRequest listenerThreaded(boolean threadedListener) { + super.listenerThreaded(threadedListener); + return this; + } + + public ValidateRequest indices(String... indices) { + this.indices = indices; + return this; + } + + /** + * The query source to execute. + */ + byte[] querySource() { + return querySource; + } + + int querySourceOffset() { + return querySourceOffset; + } + + int querySourceLength() { + return querySourceLength; + } + + /** + * The query source to execute. + * + * @see org.elasticsearch.index.query.QueryBuilders + */ + @Required + public ValidateRequest query(QueryBuilder queryBuilder) { + BytesStream bos = queryBuilder.buildAsUnsafeBytes(); + this.querySource = bos.underlyingBytes(); + this.querySourceOffset = 0; + this.querySourceLength = bos.size(); + this.querySourceUnsafe = true; + return this; + } + + /** + * The query source to execute in the form of a map. + */ + @Required + public ValidateRequest query(Map querySource) { + try { + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + builder.map(querySource); + return query(builder); + } catch (IOException e) { + throw new ElasticSearchGenerationException("Failed to generate [" + querySource + "]", e); + } + } + + @Required + public ValidateRequest query(XContentBuilder builder) { + try { + this.querySource = builder.underlyingBytes(); + this.querySourceOffset = 0; + this.querySourceLength = builder.underlyingBytesLength(); + this.querySourceUnsafe = false; + return this; + } catch (IOException e) { + throw new ElasticSearchGenerationException("Failed to generate [" + builder + "]", e); + } + } + + /** + * The query source to validate. It is preferable to use either {@link #query(byte[])} + * or {@link #query(org.elasticsearch.index.query.QueryBuilder)}. + */ + @Required + public ValidateRequest query(String querySource) { + UnicodeUtil.UTF8Result result = Unicode.fromStringAsUtf8(querySource); + this.querySource = result.result; + this.querySourceOffset = 0; + this.querySourceLength = result.length; + this.querySourceUnsafe = true; + return this; + } + + /** + * The query source to validate. + */ + @Required + public ValidateRequest query(byte[] querySource) { + return query(querySource, 0, querySource.length, false); + } + + /** + * The query source to validate. + */ + @Required + public ValidateRequest query(byte[] querySource, int offset, int length, boolean unsafe) { + this.querySource = querySource; + this.querySourceOffset = offset; + this.querySourceLength = length; + this.querySourceUnsafe = unsafe; + return this; + } + + /** + * The types of documents the query will run against. Defaults to all types. + */ + String[] types() { + return this.types; + } + + /** + * The types of documents the query will run against. Defaults to all types. + */ + public ValidateRequest types(String... types) { + this.types = types; + return this; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + + querySourceUnsafe = false; + querySourceOffset = 0; + querySourceLength = in.readVInt(); + querySource = new byte[querySourceLength]; + in.readFully(querySource); + + int typesSize = in.readVInt(); + if (typesSize > 0) { + types = new String[typesSize]; + for (int i = 0; i < typesSize; i++) { + types[i] = in.readUTF(); + } + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + + out.writeVInt(querySourceLength); + out.writeBytes(querySource, querySourceOffset, querySourceLength); + + out.writeVInt(types.length); + for (String type : types) { + out.writeUTF(type); + } + } + + @Override + public String toString() { + return "[" + Arrays.toString(indices) + "]" + Arrays.toString(types) + ", querySource[" + Unicode.fromBytes(querySource) + "]"; + } +} diff --git a/src/main/java/org/elasticsearch/action/validate/ValidateResponse.java b/src/main/java/org/elasticsearch/action/validate/ValidateResponse.java new file mode 100644 index 00000000000..590b7583100 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/validate/ValidateResponse.java @@ -0,0 +1,73 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.validate; + +import org.elasticsearch.action.ShardOperationFailedException; +import org.elasticsearch.action.support.broadcast.BroadcastOperationResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.List; + +/** + * The response of the validate action. + * + * + */ +public class ValidateResponse extends BroadcastOperationResponse { + + private boolean valid; + + ValidateResponse() { + + } + + ValidateResponse(boolean valid, int totalShards, int successfulShards, int failedShards, List shardFailures) { + super(totalShards, successfulShards, failedShards, shardFailures); + this.valid = valid; + } + + /** + * A boolean denoting whether the query is valid. + */ + public boolean valid() { + return valid; + } + + /** + * A boolean denoting whether the query is valid. + */ + public boolean getValid() { + return valid; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + valid = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBoolean(valid); + } +} diff --git a/src/main/java/org/elasticsearch/action/validate/pacakge-info.java b/src/main/java/org/elasticsearch/action/validate/pacakge-info.java new file mode 100644 index 00000000000..a6fcb8f9a49 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/validate/pacakge-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Validate action. + */ +package org.elasticsearch.action.validate; \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/Client.java b/src/main/java/org/elasticsearch/client/Client.java index 18050ca8aa1..3af762d8e9a 100644 --- a/src/main/java/org/elasticsearch/client/Client.java +++ b/src/main/java/org/elasticsearch/client/Client.java @@ -41,6 +41,8 @@ import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.action.validate.ValidateRequest; +import org.elasticsearch.action.validate.ValidateResponse; import org.elasticsearch.client.action.bulk.BulkRequestBuilder; import org.elasticsearch.client.action.count.CountRequestBuilder; import org.elasticsearch.client.action.delete.DeleteRequestBuilder; @@ -52,6 +54,7 @@ import org.elasticsearch.client.action.mlt.MoreLikeThisRequestBuilder; import org.elasticsearch.client.action.percolate.PercolateRequestBuilder; import org.elasticsearch.client.action.search.SearchRequestBuilder; import org.elasticsearch.client.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.client.action.validate.ValidateRequestBuilder; import org.elasticsearch.common.Nullable; /** @@ -274,6 +277,29 @@ public interface Client { */ CountRequestBuilder prepareCount(String... indices); + /** + * A count of all the documents matching a specific query. + * + * @param request The count request + * @return The result future + * @see Requests#countRequest(String...) + */ + ActionFuture validate(ValidateRequest request); + + /** + * A count of all the documents matching a specific query. + * + * @param request The count request + * @param listener A listener to be notified of the result + * @see Requests#countRequest(String...) + */ + void validate(ValidateRequest request, ActionListener listener); + + /** + * A count of all the documents matching a specific query. + */ + ValidateRequestBuilder prepareValidate(String... indices); + /** * Search across one or more indices and one or more types with a query. * diff --git a/src/main/java/org/elasticsearch/client/action/validate/ValidateRequestBuilder.java b/src/main/java/org/elasticsearch/client/action/validate/ValidateRequestBuilder.java new file mode 100644 index 00000000000..57c2a7cdf43 --- /dev/null +++ b/src/main/java/org/elasticsearch/client/action/validate/ValidateRequestBuilder.java @@ -0,0 +1,75 @@ +package org.elasticsearch.client.action.validate; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; +import org.elasticsearch.action.validate.ValidateRequest; +import org.elasticsearch.action.validate.ValidateResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.action.support.BaseRequestBuilder; +import org.elasticsearch.index.query.QueryBuilder; + +/** + * + */ +public class ValidateRequestBuilder extends BaseRequestBuilder { + public ValidateRequestBuilder(Client client) { + super(client, new ValidateRequest()); + } + + /** + * Sets the indices the query validation will run against. + */ + public ValidateRequestBuilder setIndices(String... indices) { + request.indices(indices); + return this; + } + + /** + * The types of documents the query will run against. Defaults to all types. + */ + public ValidateRequestBuilder setTypes(String... types) { + request.types(types); + return this; + } + + /** + * The query source to validate. + * + * @see org.elasticsearch.index.query.QueryBuilders + */ + public ValidateRequestBuilder setQuery(QueryBuilder queryBuilder) { + request.query(queryBuilder); + return this; + } + + /** + * The query source to validate. + * + * @see org.elasticsearch.index.query.QueryBuilders + */ + public ValidateRequestBuilder setQuery(byte[] querySource) { + request.query(querySource); + return this; + } + + /** + * Controls the operation threading model. + */ + public ValidateRequestBuilder setOperationThreading(BroadcastOperationThreading operationThreading) { + request.operationThreading(operationThreading); + return this; + } + + /** + * Should the listener be called on a separate thread if needed. + */ + public ValidateRequestBuilder setListenerThreaded(boolean threadedListener) { + request.listenerThreaded(threadedListener); + return this; + } + + @Override + protected void doExecute(ActionListener listener) { + client.validate(request, listener); + } +} diff --git a/src/main/java/org/elasticsearch/client/node/NodeClient.java b/src/main/java/org/elasticsearch/client/node/NodeClient.java index 74812963a95..9fb4acd4789 100644 --- a/src/main/java/org/elasticsearch/client/node/NodeClient.java +++ b/src/main/java/org/elasticsearch/client/node/NodeClient.java @@ -43,7 +43,11 @@ import org.elasticsearch.action.percolate.PercolateRequest; import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.percolate.TransportPercolateAction; import org.elasticsearch.action.search.*; +import org.elasticsearch.action.validate.TransportValidateAction; +import org.elasticsearch.action.validate.ValidateRequest; +import org.elasticsearch.action.validate.ValidateResponse; import org.elasticsearch.client.AdminClient; +import org.elasticsearch.client.action.validate.ValidateRequestBuilder; import org.elasticsearch.client.internal.InternalClient; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.common.inject.Inject; @@ -73,6 +77,8 @@ public class NodeClient extends AbstractClient implements InternalClient { private final TransportCountAction countAction; + private final TransportValidateAction validateAction; + private final TransportSearchAction searchAction; private final TransportSearchScrollAction searchScrollAction; @@ -85,7 +91,7 @@ public class NodeClient extends AbstractClient implements InternalClient { public NodeClient(Settings settings, ThreadPool threadPool, NodeAdminClient admin, TransportIndexAction indexAction, TransportDeleteAction deleteAction, TransportBulkAction bulkAction, TransportDeleteByQueryAction deleteByQueryAction, TransportGetAction getAction, TransportMultiGetAction multiGetAction, TransportCountAction countAction, - TransportSearchAction searchAction, TransportSearchScrollAction searchScrollAction, + TransportSearchAction searchAction, TransportValidateAction validateAction, TransportSearchScrollAction searchScrollAction, TransportMoreLikeThisAction moreLikeThisAction, TransportPercolateAction percolateAction) { this.threadPool = threadPool; this.admin = admin; @@ -96,6 +102,7 @@ public class NodeClient extends AbstractClient implements InternalClient { this.getAction = getAction; this.multiGetAction = multiGetAction; this.countAction = countAction; + this.validateAction = validateAction; this.searchAction = searchAction; this.searchScrollAction = searchScrollAction; this.moreLikeThisAction = moreLikeThisAction; @@ -187,6 +194,16 @@ public class NodeClient extends AbstractClient implements InternalClient { countAction.execute(request, listener); } + @Override + public ActionFuture validate(ValidateRequest request) { + return validateAction.execute(request); + } + + @Override + public void validate(ValidateRequest request, ActionListener listener) { + validateAction.execute(request, listener); + } + @Override public ActionFuture search(SearchRequest request) { return searchAction.execute(request); diff --git a/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/src/main/java/org/elasticsearch/client/support/AbstractClient.java index 38340d8f36d..246b848e796 100644 --- a/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -30,6 +30,7 @@ import org.elasticsearch.client.action.mlt.MoreLikeThisRequestBuilder; import org.elasticsearch.client.action.percolate.PercolateRequestBuilder; import org.elasticsearch.client.action.search.SearchRequestBuilder; import org.elasticsearch.client.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.client.action.validate.ValidateRequestBuilder; import org.elasticsearch.client.internal.InternalClient; import org.elasticsearch.common.Nullable; @@ -103,6 +104,11 @@ public abstract class AbstractClient implements InternalClient { return new CountRequestBuilder(this).setIndices(indices); } + @Override + public ValidateRequestBuilder prepareValidate(String... indices) { + return new ValidateRequestBuilder(this).setIndices(indices); + } + @Override public MoreLikeThisRequestBuilder prepareMoreLikeThis(String index, String type, String id) { return new MoreLikeThisRequestBuilder(this, index, type, id); diff --git a/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/src/main/java/org/elasticsearch/client/transport/TransportClient.java index 890ad1f3b38..569b255422f 100644 --- a/src/main/java/org/elasticsearch/client/transport/TransportClient.java +++ b/src/main/java/org/elasticsearch/client/transport/TransportClient.java @@ -43,6 +43,8 @@ import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.action.validate.ValidateRequest; +import org.elasticsearch.action.validate.ValidateResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.client.transport.action.ClientTransportActionModule; @@ -319,6 +321,16 @@ public class TransportClient extends AbstractClient { internalClient.count(request, listener); } + @Override + public ActionFuture validate(ValidateRequest request) { + return internalClient.validate(request); + } + + @Override + public void validate(ValidateRequest request, ActionListener listener) { + internalClient.validate(request, listener); + } + @Override public ActionFuture search(SearchRequest request) { return internalClient.search(request); diff --git a/src/main/java/org/elasticsearch/client/transport/action/validate/ClientTransportValidateAction.java b/src/main/java/org/elasticsearch/client/transport/action/validate/ClientTransportValidateAction.java new file mode 100644 index 00000000000..d449a7e673e --- /dev/null +++ b/src/main/java/org/elasticsearch/client/transport/action/validate/ClientTransportValidateAction.java @@ -0,0 +1,25 @@ +package org.elasticsearch.client.transport.action.validate; + +import org.elasticsearch.action.TransportActions; +import org.elasticsearch.action.validate.ValidateRequest; +import org.elasticsearch.action.validate.ValidateResponse; +import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.transport.TransportService; + +/** + * + */ +public class ClientTransportValidateAction extends BaseClientTransportAction { + + @Inject + public ClientTransportValidateAction(Settings settings, TransportService transportService) { + super(settings, transportService, ValidateResponse.class); + } + + @Override + protected String action() { + return TransportActions.VALIDATE; + } +} diff --git a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java index 6f9370c4c61..96868f6c500 100644 --- a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java +++ b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java @@ -42,6 +42,8 @@ import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.action.validate.ValidateRequest; +import org.elasticsearch.action.validate.ValidateResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.internal.InternalClient; import org.elasticsearch.client.support.AbstractClient; @@ -57,6 +59,7 @@ import org.elasticsearch.client.transport.action.mlt.ClientTransportMoreLikeThis import org.elasticsearch.client.transport.action.percolate.ClientTransportPercolateAction; import org.elasticsearch.client.transport.action.search.ClientTransportSearchAction; import org.elasticsearch.client.transport.action.search.ClientTransportSearchScrollAction; +import org.elasticsearch.client.transport.action.validate.ClientTransportValidateAction; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -87,6 +90,8 @@ public class InternalTransportClient extends AbstractClient implements InternalC private final ClientTransportCountAction countAction; + private final ClientTransportValidateAction validateAction; + private final ClientTransportSearchAction searchAction; private final ClientTransportSearchScrollAction searchScrollAction; @@ -99,7 +104,7 @@ public class InternalTransportClient extends AbstractClient implements InternalC public InternalTransportClient(Settings settings, ThreadPool threadPool, TransportClientNodesService nodesService, InternalTransportAdminClient adminClient, ClientTransportIndexAction indexAction, ClientTransportDeleteAction deleteAction, ClientTransportBulkAction bulkAction, ClientTransportGetAction getAction, ClientTransportMultiGetAction multiGetAction, - ClientTransportDeleteByQueryAction deleteByQueryAction, ClientTransportCountAction countAction, + ClientTransportDeleteByQueryAction deleteByQueryAction, ClientTransportCountAction countAction, ClientTransportValidateAction validateAction, ClientTransportSearchAction searchAction, ClientTransportSearchScrollAction searchScrollAction, ClientTransportMoreLikeThisAction moreLikeThisAction, ClientTransportPercolateAction percolateAction) { this.threadPool = threadPool; @@ -113,6 +118,7 @@ public class InternalTransportClient extends AbstractClient implements InternalC this.multiGetAction = multiGetAction; this.deleteByQueryAction = deleteByQueryAction; this.countAction = countAction; + this.validateAction = validateAction; this.searchAction = searchAction; this.searchScrollAction = searchScrollAction; this.moreLikeThisAction = moreLikeThisAction; @@ -274,6 +280,26 @@ public class InternalTransportClient extends AbstractClient implements InternalC }, listener); } + @Override + public ActionFuture validate(final ValidateRequest request) { + return nodesService.execute(new TransportClientNodesService.NodeCallback>() { + @Override + public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { + return validateAction.execute(node, request); + } + }); + } + + @Override + public void validate(final ValidateRequest request, final ActionListener listener) { + nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { + @Override + public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { + validateAction.execute(node, request, listener); + } + }, listener); + } + @Override public ActionFuture search(final SearchRequest request) { return nodesService.execute(new TransportClientNodesService.NodeCallback>() { diff --git a/src/main/java/org/elasticsearch/index/shard/service/IndexShard.java b/src/main/java/org/elasticsearch/index/shard/service/IndexShard.java index d39c6eb9fd1..c51b6dbe985 100644 --- a/src/main/java/org/elasticsearch/index/shard/service/IndexShard.java +++ b/src/main/java/org/elasticsearch/index/shard/service/IndexShard.java @@ -105,6 +105,10 @@ public interface IndexShard extends IndexShardComponent { Engine.Searcher searcher(); + boolean validate(byte[] querySource, @Nullable String[] filteringAliases, String... types) throws ElasticSearchException; + + boolean validate(byte[] querySource, int querySourceOffset, int querySourceLength, @Nullable String[] filteringAliases, String... types) throws ElasticSearchException; + /** * Returns true if this shard can ignore a recovery attempt made to it (since the already doing/done it) */ diff --git a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java index 7a82e0d0673..9a180776d36 100644 --- a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java +++ b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java @@ -51,6 +51,7 @@ import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.merge.scheduler.MergeSchedulerProvider; import org.elasticsearch.index.query.IndexQueryParserService; +import org.elasticsearch.index.query.QueryParsingException; import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.search.stats.ShardSearchService; @@ -518,6 +519,28 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I return engine.searcher(); } + @Override + public boolean validate(byte[] querySource, @Nullable String[] filteringAliases, String... types) throws ElasticSearchException { + return validate(querySource, 0, querySource.length, filteringAliases, types); + } + + @Override + public boolean validate(byte[] querySource, int querySourceOffset, int querySourceLength, @Nullable String[] filteringAliases, String... types) throws ElasticSearchException { + readAllowed(); + if (querySourceLength == 0) { + return true; + } else { + try { + queryParserService.parse(querySource, querySourceOffset, querySourceLength); + } catch (QueryParsingException e) { + return false; + } catch (AssertionError e) { + return false; + } + } + return true; + } + public void close(String reason) { synchronized (mutex) { indexSettingsService.removeListener(applyRefreshSettings); diff --git a/src/main/java/org/elasticsearch/rest/action/RestActionModule.java b/src/main/java/org/elasticsearch/rest/action/RestActionModule.java index d899a59ad89..0c10ef62803 100644 --- a/src/main/java/org/elasticsearch/rest/action/RestActionModule.java +++ b/src/main/java/org/elasticsearch/rest/action/RestActionModule.java @@ -70,6 +70,7 @@ import org.elasticsearch.rest.action.mlt.RestMoreLikeThisAction; import org.elasticsearch.rest.action.percolate.RestPercolateAction; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.rest.action.search.RestSearchScrollAction; +import org.elasticsearch.rest.action.validate.RestValidateAction; import java.util.List; @@ -149,6 +150,8 @@ public class RestActionModule extends AbstractModule { bind(RestSearchAction.class).asEagerSingleton(); bind(RestSearchScrollAction.class).asEagerSingleton(); + + bind(RestValidateAction.class).asEagerSingleton(); bind(RestMoreLikeThisAction.class).asEagerSingleton(); diff --git a/src/main/java/org/elasticsearch/rest/action/validate/RestValidateAction.java b/src/main/java/org/elasticsearch/rest/action/validate/RestValidateAction.java new file mode 100644 index 00000000000..2bbac22329d --- /dev/null +++ b/src/main/java/org/elasticsearch/rest/action/validate/RestValidateAction.java @@ -0,0 +1,122 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.validate; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; +import org.elasticsearch.action.validate.ValidateRequest; +import org.elasticsearch.action.validate.ValidateResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.*; +import org.elasticsearch.rest.action.support.RestActions; +import org.elasticsearch.rest.action.support.RestXContentBuilder; + +import java.io.IOException; + +import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; +import static org.elasticsearch.rest.RestStatus.OK; +import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastShardsHeader; +import static org.elasticsearch.rest.action.support.RestActions.splitTypes; + +/** + * + */ +public class RestValidateAction extends BaseRestHandler { + + @Inject + public RestValidateAction(Settings settings, Client client, RestController controller) { + super(settings, client); + controller.registerHandler(GET, "/_validate", this); + controller.registerHandler(POST, "/_validate", this); + controller.registerHandler(GET, "/{index}/_validate", this); + controller.registerHandler(POST, "/{index}/_validate", this); + controller.registerHandler(GET, "/{index}/{type}/_validate", this); + controller.registerHandler(POST, "/{index}/{type}/_validate", this); + } + + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + ValidateRequest validateRequest = new ValidateRequest(RestActions.splitIndices(request.param("index"))); + // we just send back a response, no need to fork a listener + validateRequest.listenerThreaded(false); + try { + BroadcastOperationThreading operationThreading = BroadcastOperationThreading.fromString(request.param("operation_threading"), BroadcastOperationThreading.SINGLE_THREAD); + if (operationThreading == BroadcastOperationThreading.NO_THREADS) { + // since we don't spawn, don't allow no_threads, but change it to a single thread + operationThreading = BroadcastOperationThreading.SINGLE_THREAD; + } + validateRequest.operationThreading(operationThreading); + if (request.hasContent()) { + validateRequest.query(request.contentByteArray(), request.contentByteArrayOffset(), request.contentLength(), true); + } else { + String source = request.param("source"); + if (source != null) { + validateRequest.query(source); + } else { + byte[] querySource = RestActions.parseQuerySource(request); + if (querySource != null) { + validateRequest.query(querySource); + } + } + } + validateRequest.types(splitTypes(request.param("type"))); + } catch (Exception e) { + try { + XContentBuilder builder = RestXContentBuilder.restContentBuilder(request); + channel.sendResponse(new XContentRestResponse(request, BAD_REQUEST, builder.startObject().field("error", e.getMessage()).endObject())); + } catch (IOException e1) { + logger.error("Failed to send failure response", e1); + } + return; + } + + client.validate(validateRequest, new ActionListener() { + @Override + public void onResponse(ValidateResponse response) { + try { + XContentBuilder builder = RestXContentBuilder.restContentBuilder(request); + builder.startObject(); + builder.field("valid", response.valid()); + + buildBroadcastShardsHeader(builder, response); + + builder.endObject(); + channel.sendResponse(new XContentRestResponse(request, OK, builder)); + } catch (Exception e) { + onFailure(e); + } + } + + @Override + public void onFailure(Throwable e) { + try { + channel.sendResponse(new XContentThrowableRestResponse(request, e)); + } catch (IOException e1) { + logger.error("Failed to send failure response", e1); + } + } + }); + } +} diff --git a/src/test/java/org/elasticsearch/test/integration/validate/SimpleValidateTests.java b/src/test/java/org/elasticsearch/test/integration/validate/SimpleValidateTests.java new file mode 100644 index 00000000000..b117f3abcfb --- /dev/null +++ b/src/test/java/org/elasticsearch/test/integration/validate/SimpleValidateTests.java @@ -0,0 +1,84 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.integration.validate; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.test.integration.AbstractNodesTests; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + +/** + * + */ +public class SimpleValidateTests extends AbstractNodesTests { + + private Client client; + + @BeforeClass + public void createNodes() throws Exception { + startNode("node1"); + startNode("node2"); + client = getClient(); + } + + @AfterClass + public void closeNodes() { + client.close(); + closeAllNodes(); + } + + protected Client getClient() { + return client("node1"); + } + + @Test + public void simpleValidateQuery() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + + client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1)).execute().actionGet(); + client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); + client.admin().indices().preparePutMapping("test").setType("type1") + .setSource(XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") + .startObject("foo").field("type", "string").endObject() + .startObject("bar").field("type", "integer").endObject() + .endObject().endObject().endObject()) + .execute().actionGet(); + + client.admin().indices().prepareRefresh().execute().actionGet(); + + assertThat(client.prepareValidate("test").setQuery("foo".getBytes()).execute().actionGet().valid(), equalTo(false)); + assertThat(client.prepareValidate("test").setQuery(QueryBuilders.queryString("_id:1")).execute().actionGet().valid(), equalTo(true)); + assertThat(client.prepareValidate("test").setQuery(QueryBuilders.queryString("_i:d:1")).execute().actionGet().valid(), equalTo(false)); + + assertThat(client.prepareValidate("test").setQuery(QueryBuilders.queryString("foo:1")).execute().actionGet().valid(), equalTo(true)); + assertThat(client.prepareValidate("test").setQuery(QueryBuilders.queryString("bar:hey")).execute().actionGet().valid(), equalTo(false)); + + assertThat(client.prepareValidate("test").setQuery(QueryBuilders.queryString("nonexistent:hello")).execute().actionGet().valid(), equalTo(true)); + + assertThat(client.prepareValidate("test").setQuery(QueryBuilders.queryString("foo:1 AND")).execute().actionGet().valid(), equalTo(false)); + } +} From 0cb1a2ebe1db6fde1621c926e7dba17abfbe99f0 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 28 Dec 2011 08:54:55 -0700 Subject: [PATCH 082/270] Prefer local shards for validate action --- .../elasticsearch/action/validate/TransportValidateAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/action/validate/TransportValidateAction.java b/src/main/java/org/elasticsearch/action/validate/TransportValidateAction.java index d17771847ff..3a91535d034 100644 --- a/src/main/java/org/elasticsearch/action/validate/TransportValidateAction.java +++ b/src/main/java/org/elasticsearch/action/validate/TransportValidateAction.java @@ -97,7 +97,7 @@ public class TransportValidateAction extends TransportBroadcastOperationAction> routingMap = clusterState.metaData().resolveSearchRouting("0", request.indices()); - return clusterService.operationRouting().searchShards(clusterState, request.indices(), concreteIndices, null, routingMap, null); + return clusterService.operationRouting().searchShards(clusterState, request.indices(), concreteIndices, null, routingMap, "_local"); } @Override From 636f7bd4546444f3d17c4f170f4ac7b899ad9f93 Mon Sep 17 00:00:00 2001 From: Edward Wang Date: Thu, 29 Dec 2011 03:47:37 +0800 Subject: [PATCH 083/270] Fix failed test ClusterAndIndexReaderOnlyTests MetaData uses two different string constants, "cluster.blocks.read_only" and "cluster.read_only", for one setting. It confuses TransportClusterUpdateSettingsAction and causes lusterAndIndexReaderOnlyTests never finish. --- src/main/java/org/elasticsearch/cluster/metadata/MetaData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 916637fd61b..cdc6728ec93 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -54,7 +54,7 @@ public class MetaData implements Iterable { public static final ClusterBlock CLUSTER_READ_ONLY_BLOCK = new ClusterBlock(6, "cluster read-only (api)", false, false, ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA); private static ImmutableSet dynamicSettings = ImmutableSet.builder() - .add("cluster.read_only") + .add(SETTING_READ_ONLY) .build(); public static ImmutableSet dynamicSettings() { From cb0284f8021481c8b8988b04bae1cdc29e85d7b8 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 28 Dec 2011 22:06:13 +0200 Subject: [PATCH 084/270] count down latch if we don't have any changes in cluster update settings --- .../cluster/settings/TransportClusterUpdateSettingsAction.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java index 85482257de9..0d6d5e847fb 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java @@ -109,6 +109,7 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeOpe } if (!changed) { + latch.countDown(); return currentState; } From bb63d38a659dcb5711f4ab4d1db22bd0f4c2bc52 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 28 Dec 2011 22:20:09 +0200 Subject: [PATCH 085/270] Two different indexes with the same nested object name causing ArrayIndexOutOfBoundsException, closes #1575. --- .../elasticsearch/index/query/NestedFilterParser.java | 9 +++++---- .../org/elasticsearch/index/query/NestedQueryParser.java | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/query/NestedFilterParser.java b/src/main/java/org/elasticsearch/index/query/NestedFilterParser.java index 5aab30e927c..7da8860da52 100644 --- a/src/main/java/org/elasticsearch/index/query/NestedFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/NestedFilterParser.java @@ -125,10 +125,11 @@ public class NestedFilterParser implements FilterParser { Filter parentFilter = currentParentFilterContext; if (parentFilter == null) { parentFilter = NonNestedDocsFilter.INSTANCE; - if (mapper.hasDocMapper()) { - // filter based on the type... - parentFilter = mapper.docMapper().typeFilter(); - } + // don't do special parent filtering, since we might have same nested mapping on two different types + //if (mapper.hasDocMapper()) { + // // filter based on the type... + // parentFilter = mapper.docMapper().typeFilter(); + //} parentFilter = parseContext.cacheFilter(parentFilter, null); } diff --git a/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java b/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java index b5d2d0c431b..da30dcd7bac 100644 --- a/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java @@ -131,10 +131,11 @@ public class NestedQueryParser implements QueryParser { Filter parentFilter = currentParentFilterContext; if (parentFilter == null) { parentFilter = NonNestedDocsFilter.INSTANCE; - if (mapper.hasDocMapper()) { - // filter based on the type... - parentFilter = mapper.docMapper().typeFilter(); - } + // don't do special parent filtering, since we might have same nested mapping on two different types + //if (mapper.hasDocMapper()) { + // // filter based on the type... + // parentFilter = mapper.docMapper().typeFilter(); + //} parentFilter = parseContext.cacheFilter(parentFilter, null); } From 4e6217c54de37f99188666ff7f43959fa1529630 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 28 Dec 2011 23:35:04 +0200 Subject: [PATCH 086/270] simplify toString for cached filter --- .../AbstractConcurrentMapFilterCache.java | 4 +--- .../support/AbstractWeightedFilterCache.java | 2 +- .../aliases/IndexAliasesServiceTests.java | 21 ++++++++++++------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java b/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java index 4826b9269ed..f68ab433331 100644 --- a/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java +++ b/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java @@ -38,8 +38,6 @@ import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.new /** * A base concurrent filter cache that accepts the actual cache to use. - * - * */ public abstract class AbstractConcurrentMapFilterCache extends AbstractIndexComponent implements FilterCache, IndexReader.ReaderFinishedListener { @@ -163,7 +161,7 @@ public abstract class AbstractConcurrentMapFilterCache extends AbstractIndexComp } public String toString() { - return "FilterCacheFilterWrapper(" + filter + ")"; + return "cache(" + filter + ")"; } public boolean equals(Object o) { diff --git a/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractWeightedFilterCache.java b/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractWeightedFilterCache.java index 1389e12c33b..f0b50c3b7bc 100644 --- a/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractWeightedFilterCache.java +++ b/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractWeightedFilterCache.java @@ -173,7 +173,7 @@ public abstract class AbstractWeightedFilterCache extends AbstractIndexComponent } public String toString() { - return "FilterCacheFilterWrapper(" + filter + ")"; + return "cache(" + filter + ")"; } public boolean equals(Object o) { diff --git a/src/test/java/org/elasticsearch/test/unit/index/aliases/IndexAliasesServiceTests.java b/src/test/java/org/elasticsearch/test/unit/index/aliases/IndexAliasesServiceTests.java index 9b9245e1703..23a65f6d8ae 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/aliases/IndexAliasesServiceTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/aliases/IndexAliasesServiceTests.java @@ -90,8 +90,8 @@ public class IndexAliasesServiceTests { assertThat(indexAliasesService.hasAlias("dogs"), equalTo(true)); assertThat(indexAliasesService.hasAlias("turtles"), equalTo(false)); - assertThat(indexAliasesService.aliasFilter("cats").toString(), equalTo("FilterCacheFilterWrapper(animal:cat)")); - assertThat(indexAliasesService.aliasFilter("cats", "dogs").toString(), equalTo("BooleanFilter( FilterCacheFilterWrapper(animal:cat) FilterCacheFilterWrapper(animal:dog))")); + assertThat(indexAliasesService.aliasFilter("cats").toString(), equalTo("cache(animal:cat)")); + assertThat(indexAliasesService.aliasFilter("cats", "dogs").toString(), equalTo("BooleanFilter( cache(animal:cat) cache(animal:dog))")); // Non-filtering alias should turn off all filters because filters are ORed assertThat(indexAliasesService.aliasFilter("all"), nullValue()); @@ -100,7 +100,7 @@ public class IndexAliasesServiceTests { indexAliasesService.add("cats", filter(termFilter("animal", "feline"))); indexAliasesService.add("dogs", filter(termFilter("animal", "canine"))); - assertThat(indexAliasesService.aliasFilter("dogs", "cats").toString(), equalTo("BooleanFilter( FilterCacheFilterWrapper(animal:canine) FilterCacheFilterWrapper(animal:feline))")); + assertThat(indexAliasesService.aliasFilter("dogs", "cats").toString(), equalTo("BooleanFilter( cache(animal:canine) cache(animal:feline))")); } @Test @@ -110,13 +110,13 @@ public class IndexAliasesServiceTests { indexAliasesService.add("dogs", filter(termFilter("animal", "dog"))); assertThat(indexAliasesService.aliasFilter(), nullValue()); - assertThat(indexAliasesService.aliasFilter("dogs").toString(), equalTo("FilterCacheFilterWrapper(animal:dog)")); - assertThat(indexAliasesService.aliasFilter("dogs", "cats").toString(), equalTo("BooleanFilter( FilterCacheFilterWrapper(animal:dog) FilterCacheFilterWrapper(animal:cat))")); + assertThat(indexAliasesService.aliasFilter("dogs").toString(), equalTo("cache(animal:dog)")); + assertThat(indexAliasesService.aliasFilter("dogs", "cats").toString(), equalTo("BooleanFilter( cache(animal:dog) cache(animal:cat))")); indexAliasesService.add("cats", filter(termFilter("animal", "feline"))); indexAliasesService.add("dogs", filter(termFilter("animal", "canine"))); - assertThat(indexAliasesService.aliasFilter("dogs", "cats").toString(), equalTo("BooleanFilter( FilterCacheFilterWrapper(animal:canine) FilterCacheFilterWrapper(animal:feline))")); + assertThat(indexAliasesService.aliasFilter("dogs", "cats").toString(), equalTo("BooleanFilter( cache(animal:canine) cache(animal:feline))")); } @Test(expectedExceptions = InvalidAliasNameException.class) @@ -128,13 +128,18 @@ public class IndexAliasesServiceTests { } - @Test(expectedExceptions = InvalidAliasNameException.class) + @Test public void testUnknownAliasFilter() throws Exception { IndexAliasesService indexAliasesService = newIndexAliasesService(); indexAliasesService.add("cats", filter(termFilter("animal", "cat"))); indexAliasesService.add("dogs", filter(termFilter("animal", "dog"))); - indexAliasesService.aliasFilter("unknown"); + try { + indexAliasesService.aliasFilter("unknown"); + assert false; + } catch (InvalidAliasNameException e) { + // all is well + } } From f6b036f713956448c040be1e1a2525b85858da13 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 28 Dec 2011 15:24:08 -0700 Subject: [PATCH 087/270] Refactor validate to validateQuery and move into indices admin action --- .../action/TransportActions.java | 3 +- .../query/ShardValidateQueryRequest.java} | 8 +- .../query/ShardValidateQueryResponse.java} | 8 +- .../query/TransportValidateQueryAction.java} | 59 +++++++++------ .../validate/query/ValidateQueryRequest.java} | 29 ++++--- .../query/ValidateQueryResponse.java} | 8 +- .../indices/validate/query/package-info.java} | 2 +- .../java/org/elasticsearch/client/Client.java | 26 ------- .../client/IndicesAdminClient.java | 26 +++++++ .../query/ValidateQueryRequestBuilder.java | 75 +++++++++++++++++++ .../validate/ValidateRequestBuilder.java | 75 ------------------- .../elasticsearch/client/node/NodeClient.java | 19 +---- .../client/node/NodeIndicesAdminClient.java | 18 ++++- .../client/support/AbstractClient.java | 6 -- .../support/AbstractIndicesAdminClient.java | 6 ++ .../client/transport/TransportClient.java | 12 --- .../ClientTransportValidateQueryAction.java | 25 +++++++ .../ClientTransportValidateAction.java | 25 ------- .../support/InternalTransportClient.java | 28 +------ .../InternalTransportIndicesAdminClient.java | 28 ++++++- .../index/shard/service/IndexShard.java | 4 - .../shard/service/InternalIndexShard.java | 23 ------ .../rest/action/RestActionModule.java | 4 +- .../query/RestValidateQueryAction.java} | 40 +++++----- ...sts.java => SimpleValidateQueryTests.java} | 16 ++-- 25 files changed, 272 insertions(+), 301 deletions(-) rename src/main/java/org/elasticsearch/action/{validate/ShardValidateRequest.java => admin/indices/validate/query/ShardValidateQueryRequest.java} (91%) rename src/main/java/org/elasticsearch/action/{validate/ShardValidateResponse.java => admin/indices/validate/query/ShardValidateQueryResponse.java} (86%) rename src/main/java/org/elasticsearch/action/{validate/TransportValidateAction.java => admin/indices/validate/query/TransportValidateQueryAction.java} (60%) rename src/main/java/org/elasticsearch/action/{validate/ValidateRequest.java => admin/indices/validate/query/ValidateQueryRequest.java} (88%) rename src/main/java/org/elasticsearch/action/{validate/ValidateResponse.java => admin/indices/validate/query/ValidateQueryResponse.java} (85%) rename src/main/java/org/elasticsearch/action/{validate/pacakge-info.java => admin/indices/validate/query/package-info.java} (93%) create mode 100644 src/main/java/org/elasticsearch/client/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java delete mode 100644 src/main/java/org/elasticsearch/client/action/validate/ValidateRequestBuilder.java create mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/validate/query/ClientTransportValidateQueryAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/validate/ClientTransportValidateAction.java rename src/main/java/org/elasticsearch/rest/action/{validate/RestValidateAction.java => admin/indices/validate/query/RestValidateQueryAction.java} (74%) rename src/test/java/org/elasticsearch/test/integration/validate/{SimpleValidateTests.java => SimpleValidateQueryTests.java} (68%) diff --git a/src/main/java/org/elasticsearch/action/TransportActions.java b/src/main/java/org/elasticsearch/action/TransportActions.java index f3826e17405..a0c7ab46851 100644 --- a/src/main/java/org/elasticsearch/action/TransportActions.java +++ b/src/main/java/org/elasticsearch/action/TransportActions.java @@ -29,8 +29,6 @@ public class TransportActions { public static final String INDEX = "indices/index/shard/index"; public static final String COUNT = "indices/count"; - - public static final String VALIDATE = "indices/validate"; public static final String DELETE = "indices/index/shard/delete"; @@ -69,6 +67,7 @@ public class TransportActions { public static final String ANALYZE = "indices/analyze"; public static final String PUT_INDEX_TEMPLATE = "indices/putIndexTemplate"; public static final String DELETE_INDEX_TEMPLATE = "indices/deleteIndexTemplate"; + public static final String VALIDATE_QUERY = "indices/validateQuery"; public static class Gateway { public static final String SNAPSHOT = "indices/gateway/snapshot"; diff --git a/src/main/java/org/elasticsearch/action/validate/ShardValidateRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java similarity index 91% rename from src/main/java/org/elasticsearch/action/validate/ShardValidateRequest.java rename to src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java index d275a43c706..6978bbbb1f8 100644 --- a/src/main/java/org/elasticsearch/action/validate/ShardValidateRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.validate; +package org.elasticsearch.action.admin.indices.validate.query; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationRequest; import org.elasticsearch.common.Nullable; @@ -32,7 +32,7 @@ import java.io.IOException; * * */ -class ShardValidateRequest extends BroadcastShardOperationRequest { +class ShardValidateQueryRequest extends BroadcastShardOperationRequest { private byte[] querySource; private int querySourceOffset; @@ -43,11 +43,11 @@ class ShardValidateRequest extends BroadcastShardOperationRequest { @Nullable private String[] filteringAliases; - ShardValidateRequest() { + ShardValidateQueryRequest() { } - public ShardValidateRequest(String index, int shardId, @Nullable String[] filteringAliases, ValidateRequest request) { + public ShardValidateQueryRequest(String index, int shardId, @Nullable String[] filteringAliases, ValidateQueryRequest request) { super(index, shardId); this.querySource = request.querySource(); this.querySourceOffset = request.querySourceOffset(); diff --git a/src/main/java/org/elasticsearch/action/validate/ShardValidateResponse.java b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryResponse.java similarity index 86% rename from src/main/java/org/elasticsearch/action/validate/ShardValidateResponse.java rename to src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryResponse.java index 31b16cddcc7..e1f00448bc7 100644 --- a/src/main/java/org/elasticsearch/action/validate/ShardValidateResponse.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryResponse.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.validate; +package org.elasticsearch.action.admin.indices.validate.query; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationResponse; import org.elasticsearch.common.io.stream.StreamInput; @@ -30,15 +30,15 @@ import java.io.IOException; * * */ -class ShardValidateResponse extends BroadcastShardOperationResponse { +class ShardValidateQueryResponse extends BroadcastShardOperationResponse { private boolean valid; - ShardValidateResponse() { + ShardValidateQueryResponse() { } - public ShardValidateResponse(String index, int shardId, boolean valid) { + public ShardValidateQueryResponse(String index, int shardId, boolean valid) { super(index, shardId); this.valid = valid; } diff --git a/src/main/java/org/elasticsearch/action/validate/TransportValidateAction.java b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java similarity index 60% rename from src/main/java/org/elasticsearch/action/validate/TransportValidateAction.java rename to src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java index 3a91535d034..6ffd48c8275 100644 --- a/src/main/java/org/elasticsearch/action/validate/TransportValidateAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.validate; +package org.elasticsearch.action.admin.indices.validate.query; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ShardOperationFailedException; @@ -32,6 +32,8 @@ import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.IndexQueryParserService; +import org.elasticsearch.index.query.QueryParsingException; import org.elasticsearch.index.shard.service.IndexShard; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.threadpool.ThreadPool; @@ -47,12 +49,12 @@ import static com.google.common.collect.Lists.newArrayList; /** * */ -public class TransportValidateAction extends TransportBroadcastOperationAction { +public class TransportValidateQueryAction extends TransportBroadcastOperationAction { private final IndicesService indicesService; @Inject - public TransportValidateAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, IndicesService indicesService) { + public TransportValidateQueryAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, IndicesService indicesService) { super(settings, threadPool, clusterService, transportService); this.indicesService = indicesService; } @@ -64,51 +66,51 @@ public class TransportValidateAction extends TransportBroadcastOperationAction> routingMap = clusterState.metaData().resolveSearchRouting("0", request.indices()); return clusterService.operationRouting().searchShards(clusterState, request.indices(), concreteIndices, null, routingMap, "_local"); } @Override - protected void checkBlock(ValidateRequest request, String[] concreteIndices, ClusterState state) { + protected void checkBlock(ValidateQueryRequest request, String[] concreteIndices, ClusterState state) { for (String index : concreteIndices) { state.blocks().indexBlocked(ClusterBlockLevel.READ, index); } } @Override - protected ValidateResponse newResponse(ValidateRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) { + protected ValidateQueryResponse newResponse(ValidateQueryRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) { int successfulShards = 0; int failedShards = 0; boolean valid = true; @@ -124,18 +126,29 @@ public class TransportValidateAction extends TransportBroadcastOperationAction shardFailures) { + ValidateQueryResponse(boolean valid, int totalShards, int successfulShards, int failedShards, List shardFailures) { super(totalShards, successfulShards, failedShards, shardFailures); this.valid = valid; } diff --git a/src/main/java/org/elasticsearch/action/validate/pacakge-info.java b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/package-info.java similarity index 93% rename from src/main/java/org/elasticsearch/action/validate/pacakge-info.java rename to src/main/java/org/elasticsearch/action/admin/indices/validate/query/package-info.java index a6fcb8f9a49..9ed901aaaf3 100644 --- a/src/main/java/org/elasticsearch/action/validate/pacakge-info.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/package-info.java @@ -20,4 +20,4 @@ /** * Validate action. */ -package org.elasticsearch.action.validate; \ No newline at end of file +package org.elasticsearch.action.admin.indices.validate.query; \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/Client.java b/src/main/java/org/elasticsearch/client/Client.java index 3af762d8e9a..18050ca8aa1 100644 --- a/src/main/java/org/elasticsearch/client/Client.java +++ b/src/main/java/org/elasticsearch/client/Client.java @@ -41,8 +41,6 @@ import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; -import org.elasticsearch.action.validate.ValidateRequest; -import org.elasticsearch.action.validate.ValidateResponse; import org.elasticsearch.client.action.bulk.BulkRequestBuilder; import org.elasticsearch.client.action.count.CountRequestBuilder; import org.elasticsearch.client.action.delete.DeleteRequestBuilder; @@ -54,7 +52,6 @@ import org.elasticsearch.client.action.mlt.MoreLikeThisRequestBuilder; import org.elasticsearch.client.action.percolate.PercolateRequestBuilder; import org.elasticsearch.client.action.search.SearchRequestBuilder; import org.elasticsearch.client.action.search.SearchScrollRequestBuilder; -import org.elasticsearch.client.action.validate.ValidateRequestBuilder; import org.elasticsearch.common.Nullable; /** @@ -277,29 +274,6 @@ public interface Client { */ CountRequestBuilder prepareCount(String... indices); - /** - * A count of all the documents matching a specific query. - * - * @param request The count request - * @return The result future - * @see Requests#countRequest(String...) - */ - ActionFuture validate(ValidateRequest request); - - /** - * A count of all the documents matching a specific query. - * - * @param request The count request - * @param listener A listener to be notified of the result - * @see Requests#countRequest(String...) - */ - void validate(ValidateRequest request, ActionListener listener); - - /** - * A count of all the documents matching a specific query. - */ - ValidateRequestBuilder prepareValidate(String... indices); - /** * Search across one or more indices and one or more types with a query. * diff --git a/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/src/main/java/org/elasticsearch/client/IndicesAdminClient.java index 6dd1d5c0e4e..323de8a8a8a 100644 --- a/src/main/java/org/elasticsearch/client/IndicesAdminClient.java +++ b/src/main/java/org/elasticsearch/client/IndicesAdminClient.java @@ -61,6 +61,8 @@ import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplat import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.elasticsearch.client.action.admin.indices.alias.IndicesAliasesRequestBuilder; import org.elasticsearch.client.action.admin.indices.analyze.AnalyzeRequestBuilder; import org.elasticsearch.client.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder; @@ -81,6 +83,7 @@ import org.elasticsearch.client.action.admin.indices.stats.IndicesStatsRequestBu import org.elasticsearch.client.action.admin.indices.status.IndicesStatusRequestBuilder; import org.elasticsearch.client.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder; import org.elasticsearch.client.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; +import org.elasticsearch.client.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.elasticsearch.common.Nullable; /** @@ -537,4 +540,27 @@ public interface IndicesAdminClient { * @param name The name of the template. */ DeleteIndexTemplateRequestBuilder prepareDeleteTemplate(String name); + + /** + * Validate a query for correctness. + * + * @param request The count request + * @return The result future + * @see Requests#countRequest(String...) + */ + ActionFuture validateQuery(ValidateQueryRequest request); + + /** + * Validate a query for correctness. + * + * @param request The count request + * @param listener A listener to be notified of the result + * @see Requests#countRequest(String...) + */ + void validateQuery(ValidateQueryRequest request, ActionListener listener); + + /** + * Validate a query for correctness. + */ + ValidateQueryRequestBuilder prepareValidateQuery(String... indices); } diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java b/src/main/java/org/elasticsearch/client/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java new file mode 100644 index 00000000000..65c3399a2af --- /dev/null +++ b/src/main/java/org/elasticsearch/client/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java @@ -0,0 +1,75 @@ +package org.elasticsearch.client.action.admin.indices.validate.query; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; +import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; +import org.elasticsearch.client.IndicesAdminClient; +import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; +import org.elasticsearch.index.query.QueryBuilder; + +/** + * + */ +public class ValidateQueryRequestBuilder extends BaseIndicesRequestBuilder { + public ValidateQueryRequestBuilder(IndicesAdminClient client) { + super(client, new ValidateQueryRequest()); + } + + /** + * Sets the indices the query validation will run against. + */ + public ValidateQueryRequestBuilder setIndices(String... indices) { + request.indices(indices); + return this; + } + + /** + * The types of documents the query will run against. Defaults to all types. + */ + public ValidateQueryRequestBuilder setTypes(String... types) { + request.types(types); + return this; + } + + /** + * The query source to validate. + * + * @see org.elasticsearch.index.query.QueryBuilders + */ + public ValidateQueryRequestBuilder setQuery(QueryBuilder queryBuilder) { + request.query(queryBuilder); + return this; + } + + /** + * The query source to validate. + * + * @see org.elasticsearch.index.query.QueryBuilders + */ + public ValidateQueryRequestBuilder setQuery(byte[] querySource) { + request.query(querySource); + return this; + } + + /** + * Controls the operation threading model. + */ + public ValidateQueryRequestBuilder setOperationThreading(BroadcastOperationThreading operationThreading) { + request.operationThreading(operationThreading); + return this; + } + + /** + * Should the listener be called on a separate thread if needed. + */ + public ValidateQueryRequestBuilder setListenerThreaded(boolean threadedListener) { + request.listenerThreaded(threadedListener); + return this; + } + + @Override + protected void doExecute(ActionListener listener) { + client.validateQuery(request, listener); + } +} diff --git a/src/main/java/org/elasticsearch/client/action/validate/ValidateRequestBuilder.java b/src/main/java/org/elasticsearch/client/action/validate/ValidateRequestBuilder.java deleted file mode 100644 index 57c2a7cdf43..00000000000 --- a/src/main/java/org/elasticsearch/client/action/validate/ValidateRequestBuilder.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.elasticsearch.client.action.validate; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; -import org.elasticsearch.action.validate.ValidateRequest; -import org.elasticsearch.action.validate.ValidateResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.support.BaseRequestBuilder; -import org.elasticsearch.index.query.QueryBuilder; - -/** - * - */ -public class ValidateRequestBuilder extends BaseRequestBuilder { - public ValidateRequestBuilder(Client client) { - super(client, new ValidateRequest()); - } - - /** - * Sets the indices the query validation will run against. - */ - public ValidateRequestBuilder setIndices(String... indices) { - request.indices(indices); - return this; - } - - /** - * The types of documents the query will run against. Defaults to all types. - */ - public ValidateRequestBuilder setTypes(String... types) { - request.types(types); - return this; - } - - /** - * The query source to validate. - * - * @see org.elasticsearch.index.query.QueryBuilders - */ - public ValidateRequestBuilder setQuery(QueryBuilder queryBuilder) { - request.query(queryBuilder); - return this; - } - - /** - * The query source to validate. - * - * @see org.elasticsearch.index.query.QueryBuilders - */ - public ValidateRequestBuilder setQuery(byte[] querySource) { - request.query(querySource); - return this; - } - - /** - * Controls the operation threading model. - */ - public ValidateRequestBuilder setOperationThreading(BroadcastOperationThreading operationThreading) { - request.operationThreading(operationThreading); - return this; - } - - /** - * Should the listener be called on a separate thread if needed. - */ - public ValidateRequestBuilder setListenerThreaded(boolean threadedListener) { - request.listenerThreaded(threadedListener); - return this; - } - - @Override - protected void doExecute(ActionListener listener) { - client.validate(request, listener); - } -} diff --git a/src/main/java/org/elasticsearch/client/node/NodeClient.java b/src/main/java/org/elasticsearch/client/node/NodeClient.java index 9fb4acd4789..74812963a95 100644 --- a/src/main/java/org/elasticsearch/client/node/NodeClient.java +++ b/src/main/java/org/elasticsearch/client/node/NodeClient.java @@ -43,11 +43,7 @@ import org.elasticsearch.action.percolate.PercolateRequest; import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.percolate.TransportPercolateAction; import org.elasticsearch.action.search.*; -import org.elasticsearch.action.validate.TransportValidateAction; -import org.elasticsearch.action.validate.ValidateRequest; -import org.elasticsearch.action.validate.ValidateResponse; import org.elasticsearch.client.AdminClient; -import org.elasticsearch.client.action.validate.ValidateRequestBuilder; import org.elasticsearch.client.internal.InternalClient; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.common.inject.Inject; @@ -77,8 +73,6 @@ public class NodeClient extends AbstractClient implements InternalClient { private final TransportCountAction countAction; - private final TransportValidateAction validateAction; - private final TransportSearchAction searchAction; private final TransportSearchScrollAction searchScrollAction; @@ -91,7 +85,7 @@ public class NodeClient extends AbstractClient implements InternalClient { public NodeClient(Settings settings, ThreadPool threadPool, NodeAdminClient admin, TransportIndexAction indexAction, TransportDeleteAction deleteAction, TransportBulkAction bulkAction, TransportDeleteByQueryAction deleteByQueryAction, TransportGetAction getAction, TransportMultiGetAction multiGetAction, TransportCountAction countAction, - TransportSearchAction searchAction, TransportValidateAction validateAction, TransportSearchScrollAction searchScrollAction, + TransportSearchAction searchAction, TransportSearchScrollAction searchScrollAction, TransportMoreLikeThisAction moreLikeThisAction, TransportPercolateAction percolateAction) { this.threadPool = threadPool; this.admin = admin; @@ -102,7 +96,6 @@ public class NodeClient extends AbstractClient implements InternalClient { this.getAction = getAction; this.multiGetAction = multiGetAction; this.countAction = countAction; - this.validateAction = validateAction; this.searchAction = searchAction; this.searchScrollAction = searchScrollAction; this.moreLikeThisAction = moreLikeThisAction; @@ -194,16 +187,6 @@ public class NodeClient extends AbstractClient implements InternalClient { countAction.execute(request, listener); } - @Override - public ActionFuture validate(ValidateRequest request) { - return validateAction.execute(request); - } - - @Override - public void validate(ValidateRequest request, ActionListener listener) { - validateAction.execute(request, listener); - } - @Override public ActionFuture search(SearchRequest request) { return searchAction.execute(request); diff --git a/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java b/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java index 8326fdc518f..a0953a22a55 100644 --- a/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java +++ b/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java @@ -81,6 +81,9 @@ import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteInd import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; import org.elasticsearch.action.admin.indices.template.put.TransportPutIndexTemplateAction; +import org.elasticsearch.action.admin.indices.validate.query.TransportValidateQueryAction; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.client.support.AbstractIndicesAdminClient; import org.elasticsearch.common.inject.Inject; @@ -134,6 +137,8 @@ public class NodeIndicesAdminClient extends AbstractIndicesAdminClient implement private final TransportDeleteIndexTemplateAction deleteIndexTemplateAction; + private final TransportValidateQueryAction validateQueryAction; + @Inject public NodeIndicesAdminClient(Settings settings, ThreadPool threadPool, TransportIndicesExistsAction indicesExistsAction, TransportIndicesStatsAction indicesStatsAction, TransportIndicesStatusAction indicesStatusAction, TransportIndicesSegmentsAction indicesSegmentsAction, TransportCreateIndexAction createIndexAction, TransportDeleteIndexAction deleteIndexAction, @@ -142,7 +147,7 @@ public class NodeIndicesAdminClient extends AbstractIndicesAdminClient implement TransportPutMappingAction putMappingAction, TransportDeleteMappingAction deleteMappingAction, TransportGatewaySnapshotAction gatewaySnapshotAction, TransportIndicesAliasesAction indicesAliasesAction, TransportClearIndicesCacheAction clearIndicesCacheAction, TransportUpdateSettingsAction updateSettingsAction, TransportAnalyzeAction analyzeAction, - TransportPutIndexTemplateAction putIndexTemplateAction, TransportDeleteIndexTemplateAction deleteIndexTemplateAction) { + TransportPutIndexTemplateAction putIndexTemplateAction, TransportDeleteIndexTemplateAction deleteIndexTemplateAction, TransportValidateQueryAction validateQueryAction) { this.threadPool = threadPool; this.indicesExistsAction = indicesExistsAction; this.indicesStatsAction = indicesStatsAction; @@ -164,6 +169,7 @@ public class NodeIndicesAdminClient extends AbstractIndicesAdminClient implement this.analyzeAction = analyzeAction; this.putIndexTemplateAction = putIndexTemplateAction; this.deleteIndexTemplateAction = deleteIndexTemplateAction; + this.validateQueryAction = validateQueryAction; } @Override @@ -370,4 +376,14 @@ public class NodeIndicesAdminClient extends AbstractIndicesAdminClient implement public void deleteTemplate(DeleteIndexTemplateRequest request, ActionListener listener) { deleteIndexTemplateAction.execute(request, listener); } + + @Override + public ActionFuture validateQuery(ValidateQueryRequest request) { + return validateQueryAction.execute(request); + } + + @Override + public void validateQuery(ValidateQueryRequest request, ActionListener listener) { + validateQueryAction.execute(request, listener); + } } diff --git a/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/src/main/java/org/elasticsearch/client/support/AbstractClient.java index 246b848e796..38340d8f36d 100644 --- a/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -30,7 +30,6 @@ import org.elasticsearch.client.action.mlt.MoreLikeThisRequestBuilder; import org.elasticsearch.client.action.percolate.PercolateRequestBuilder; import org.elasticsearch.client.action.search.SearchRequestBuilder; import org.elasticsearch.client.action.search.SearchScrollRequestBuilder; -import org.elasticsearch.client.action.validate.ValidateRequestBuilder; import org.elasticsearch.client.internal.InternalClient; import org.elasticsearch.common.Nullable; @@ -104,11 +103,6 @@ public abstract class AbstractClient implements InternalClient { return new CountRequestBuilder(this).setIndices(indices); } - @Override - public ValidateRequestBuilder prepareValidate(String... indices) { - return new ValidateRequestBuilder(this).setIndices(indices); - } - @Override public MoreLikeThisRequestBuilder prepareMoreLikeThis(String index, String type, String id) { return new MoreLikeThisRequestBuilder(this, index, type, id); diff --git a/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java b/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java index a2ebcb7c352..04356b9be77 100644 --- a/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java +++ b/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java @@ -39,6 +39,7 @@ import org.elasticsearch.client.action.admin.indices.stats.IndicesStatsRequestBu import org.elasticsearch.client.action.admin.indices.status.IndicesStatusRequestBuilder; import org.elasticsearch.client.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder; import org.elasticsearch.client.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; +import org.elasticsearch.client.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.elasticsearch.client.internal.InternalIndicesAdminClient; import org.elasticsearch.common.Nullable; @@ -151,4 +152,9 @@ public abstract class AbstractIndicesAdminClient implements InternalIndicesAdmin public DeleteIndexTemplateRequestBuilder prepareDeleteTemplate(String name) { return new DeleteIndexTemplateRequestBuilder(this, name); } + + @Override + public ValidateQueryRequestBuilder prepareValidateQuery(String... indices) { + return new ValidateQueryRequestBuilder(this).setIndices(indices); + } } diff --git a/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/src/main/java/org/elasticsearch/client/transport/TransportClient.java index 569b255422f..890ad1f3b38 100644 --- a/src/main/java/org/elasticsearch/client/transport/TransportClient.java +++ b/src/main/java/org/elasticsearch/client/transport/TransportClient.java @@ -43,8 +43,6 @@ import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; -import org.elasticsearch.action.validate.ValidateRequest; -import org.elasticsearch.action.validate.ValidateResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.client.transport.action.ClientTransportActionModule; @@ -321,16 +319,6 @@ public class TransportClient extends AbstractClient { internalClient.count(request, listener); } - @Override - public ActionFuture validate(ValidateRequest request) { - return internalClient.validate(request); - } - - @Override - public void validate(ValidateRequest request, ActionListener listener) { - internalClient.validate(request, listener); - } - @Override public ActionFuture search(SearchRequest request) { return internalClient.search(request); diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/validate/query/ClientTransportValidateQueryAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/validate/query/ClientTransportValidateQueryAction.java new file mode 100644 index 00000000000..1072b28c140 --- /dev/null +++ b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/validate/query/ClientTransportValidateQueryAction.java @@ -0,0 +1,25 @@ +package org.elasticsearch.client.transport.action.admin.indices.validate.query; + +import org.elasticsearch.action.TransportActions; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; +import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.transport.TransportService; + +/** + * + */ +public class ClientTransportValidateQueryAction extends BaseClientTransportAction { + + @Inject + public ClientTransportValidateQueryAction(Settings settings, TransportService transportService) { + super(settings, transportService, ValidateQueryResponse.class); + } + + @Override + protected String action() { + return TransportActions.Admin.Indices.VALIDATE_QUERY; + } +} diff --git a/src/main/java/org/elasticsearch/client/transport/action/validate/ClientTransportValidateAction.java b/src/main/java/org/elasticsearch/client/transport/action/validate/ClientTransportValidateAction.java deleted file mode 100644 index d449a7e673e..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/validate/ClientTransportValidateAction.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.elasticsearch.client.transport.action.validate; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.validate.ValidateRequest; -import org.elasticsearch.action.validate.ValidateResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportValidateAction extends BaseClientTransportAction { - - @Inject - public ClientTransportValidateAction(Settings settings, TransportService transportService) { - super(settings, transportService, ValidateResponse.class); - } - - @Override - protected String action() { - return TransportActions.VALIDATE; - } -} diff --git a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java index 96868f6c500..6f9370c4c61 100644 --- a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java +++ b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java @@ -42,8 +42,6 @@ import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; -import org.elasticsearch.action.validate.ValidateRequest; -import org.elasticsearch.action.validate.ValidateResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.internal.InternalClient; import org.elasticsearch.client.support.AbstractClient; @@ -59,7 +57,6 @@ import org.elasticsearch.client.transport.action.mlt.ClientTransportMoreLikeThis import org.elasticsearch.client.transport.action.percolate.ClientTransportPercolateAction; import org.elasticsearch.client.transport.action.search.ClientTransportSearchAction; import org.elasticsearch.client.transport.action.search.ClientTransportSearchScrollAction; -import org.elasticsearch.client.transport.action.validate.ClientTransportValidateAction; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -90,8 +87,6 @@ public class InternalTransportClient extends AbstractClient implements InternalC private final ClientTransportCountAction countAction; - private final ClientTransportValidateAction validateAction; - private final ClientTransportSearchAction searchAction; private final ClientTransportSearchScrollAction searchScrollAction; @@ -104,7 +99,7 @@ public class InternalTransportClient extends AbstractClient implements InternalC public InternalTransportClient(Settings settings, ThreadPool threadPool, TransportClientNodesService nodesService, InternalTransportAdminClient adminClient, ClientTransportIndexAction indexAction, ClientTransportDeleteAction deleteAction, ClientTransportBulkAction bulkAction, ClientTransportGetAction getAction, ClientTransportMultiGetAction multiGetAction, - ClientTransportDeleteByQueryAction deleteByQueryAction, ClientTransportCountAction countAction, ClientTransportValidateAction validateAction, + ClientTransportDeleteByQueryAction deleteByQueryAction, ClientTransportCountAction countAction, ClientTransportSearchAction searchAction, ClientTransportSearchScrollAction searchScrollAction, ClientTransportMoreLikeThisAction moreLikeThisAction, ClientTransportPercolateAction percolateAction) { this.threadPool = threadPool; @@ -118,7 +113,6 @@ public class InternalTransportClient extends AbstractClient implements InternalC this.multiGetAction = multiGetAction; this.deleteByQueryAction = deleteByQueryAction; this.countAction = countAction; - this.validateAction = validateAction; this.searchAction = searchAction; this.searchScrollAction = searchScrollAction; this.moreLikeThisAction = moreLikeThisAction; @@ -280,26 +274,6 @@ public class InternalTransportClient extends AbstractClient implements InternalC }, listener); } - @Override - public ActionFuture validate(final ValidateRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return validateAction.execute(node, request); - } - }); - } - - @Override - public void validate(final ValidateRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - validateAction.execute(node, request, listener); - } - }, listener); - } - @Override public ActionFuture search(final SearchRequest request) { return nodesService.execute(new TransportClientNodesService.NodeCallback>() { diff --git a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportIndicesAdminClient.java b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportIndicesAdminClient.java index 8daf3ea5aa6..12e7ce46a9e 100644 --- a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportIndicesAdminClient.java +++ b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportIndicesAdminClient.java @@ -62,6 +62,8 @@ import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplat import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.client.support.AbstractIndicesAdminClient; import org.elasticsearch.client.transport.TransportClientNodesService; @@ -85,6 +87,7 @@ import org.elasticsearch.client.transport.action.admin.indices.stats.ClientTrans import org.elasticsearch.client.transport.action.admin.indices.status.ClientTransportIndicesStatusAction; import org.elasticsearch.client.transport.action.admin.indices.template.delete.ClientTransportDeleteIndexTemplateAction; import org.elasticsearch.client.transport.action.admin.indices.template.put.ClientTransportPutIndexTemplateAction; +import org.elasticsearch.client.transport.action.admin.indices.validate.query.ClientTransportValidateQueryAction; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -139,6 +142,8 @@ public class InternalTransportIndicesAdminClient extends AbstractIndicesAdminCli private final ClientTransportDeleteIndexTemplateAction deleteIndexTemplateAction; + private final ClientTransportValidateQueryAction validateQueryAction; + @Inject public InternalTransportIndicesAdminClient(Settings settings, TransportClientNodesService nodesService, ThreadPool threadPool, ClientTransportIndicesExistsAction indicesExistsAction, ClientTransportIndicesStatusAction indicesStatusAction, ClientTransportIndicesStatsAction indicesStatsAction, ClientTransportIndicesSegmentsAction indicesSegmentsAction, @@ -147,7 +152,7 @@ public class InternalTransportIndicesAdminClient extends AbstractIndicesAdminCli ClientTransportRefreshAction refreshAction, ClientTransportFlushAction flushAction, ClientTransportOptimizeAction optimizeAction, ClientTransportPutMappingAction putMappingAction, ClientTransportDeleteMappingAction deleteMappingAction, ClientTransportGatewaySnapshotAction gatewaySnapshotAction, ClientTransportIndicesAliasesAction indicesAliasesAction, ClientTransportClearIndicesCacheAction clearIndicesCacheAction, - ClientTransportUpdateSettingsAction updateSettingsAction, ClientTransportAnalyzeAction analyzeAction, + ClientTransportUpdateSettingsAction updateSettingsAction, ClientTransportAnalyzeAction analyzeAction, ClientTransportValidateQueryAction validateQueryAction, ClientTransportPutIndexTemplateAction putIndexTemplateAction, ClientTransportDeleteIndexTemplateAction deleteIndexTemplateAction) { this.nodesService = nodesService; this.threadPool = threadPool; @@ -171,6 +176,7 @@ public class InternalTransportIndicesAdminClient extends AbstractIndicesAdminCli this.analyzeAction = analyzeAction; this.putIndexTemplateAction = putIndexTemplateAction; this.deleteIndexTemplateAction = deleteIndexTemplateAction; + this.validateQueryAction = validateQueryAction; } @Override @@ -577,4 +583,24 @@ public class InternalTransportIndicesAdminClient extends AbstractIndicesAdminCli } }, listener); } + + @Override + public ActionFuture validateQuery(final ValidateQueryRequest request) { + return nodesService.execute(new TransportClientNodesService.NodeCallback>() { + @Override + public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { + return validateQueryAction.execute(node, request); + } + }); + } + + @Override + public void validateQuery(final ValidateQueryRequest request, final ActionListener listener) { + nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { + @Override + public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { + validateQueryAction.execute(node, request, listener); + } + }, listener); + } } diff --git a/src/main/java/org/elasticsearch/index/shard/service/IndexShard.java b/src/main/java/org/elasticsearch/index/shard/service/IndexShard.java index c51b6dbe985..d39c6eb9fd1 100644 --- a/src/main/java/org/elasticsearch/index/shard/service/IndexShard.java +++ b/src/main/java/org/elasticsearch/index/shard/service/IndexShard.java @@ -105,10 +105,6 @@ public interface IndexShard extends IndexShardComponent { Engine.Searcher searcher(); - boolean validate(byte[] querySource, @Nullable String[] filteringAliases, String... types) throws ElasticSearchException; - - boolean validate(byte[] querySource, int querySourceOffset, int querySourceLength, @Nullable String[] filteringAliases, String... types) throws ElasticSearchException; - /** * Returns true if this shard can ignore a recovery attempt made to it (since the already doing/done it) */ diff --git a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java index 9a180776d36..7a82e0d0673 100644 --- a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java +++ b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java @@ -51,7 +51,6 @@ import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.merge.scheduler.MergeSchedulerProvider; import org.elasticsearch.index.query.IndexQueryParserService; -import org.elasticsearch.index.query.QueryParsingException; import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.search.stats.ShardSearchService; @@ -519,28 +518,6 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I return engine.searcher(); } - @Override - public boolean validate(byte[] querySource, @Nullable String[] filteringAliases, String... types) throws ElasticSearchException { - return validate(querySource, 0, querySource.length, filteringAliases, types); - } - - @Override - public boolean validate(byte[] querySource, int querySourceOffset, int querySourceLength, @Nullable String[] filteringAliases, String... types) throws ElasticSearchException { - readAllowed(); - if (querySourceLength == 0) { - return true; - } else { - try { - queryParserService.parse(querySource, querySourceOffset, querySourceLength); - } catch (QueryParsingException e) { - return false; - } catch (AssertionError e) { - return false; - } - } - return true; - } - public void close(String reason) { synchronized (mutex) { indexSettingsService.removeListener(applyRefreshSettings); diff --git a/src/main/java/org/elasticsearch/rest/action/RestActionModule.java b/src/main/java/org/elasticsearch/rest/action/RestActionModule.java index 0c10ef62803..b007b1738a5 100644 --- a/src/main/java/org/elasticsearch/rest/action/RestActionModule.java +++ b/src/main/java/org/elasticsearch/rest/action/RestActionModule.java @@ -58,6 +58,7 @@ import org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusActio import org.elasticsearch.rest.action.admin.indices.template.delete.RestDeleteIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.template.get.RestGetIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.template.put.RestPutIndexTemplateAction; +import org.elasticsearch.rest.action.admin.indices.validate.query.RestValidateQueryAction; import org.elasticsearch.rest.action.bulk.RestBulkAction; import org.elasticsearch.rest.action.count.RestCountAction; import org.elasticsearch.rest.action.delete.RestDeleteAction; @@ -70,7 +71,6 @@ import org.elasticsearch.rest.action.mlt.RestMoreLikeThisAction; import org.elasticsearch.rest.action.percolate.RestPercolateAction; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.rest.action.search.RestSearchScrollAction; -import org.elasticsearch.rest.action.validate.RestValidateAction; import java.util.List; @@ -151,7 +151,7 @@ public class RestActionModule extends AbstractModule { bind(RestSearchAction.class).asEagerSingleton(); bind(RestSearchScrollAction.class).asEagerSingleton(); - bind(RestValidateAction.class).asEagerSingleton(); + bind(RestValidateQueryAction.class).asEagerSingleton(); bind(RestMoreLikeThisAction.class).asEagerSingleton(); diff --git a/src/main/java/org/elasticsearch/rest/action/validate/RestValidateAction.java b/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java similarity index 74% rename from src/main/java/org/elasticsearch/rest/action/validate/RestValidateAction.java rename to src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java index 2bbac22329d..9da168c2c4c 100644 --- a/src/main/java/org/elasticsearch/rest/action/validate/RestValidateAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java @@ -17,12 +17,12 @@ * under the License. */ -package org.elasticsearch.rest.action.validate; +package org.elasticsearch.rest.action.admin.indices.validate.query; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; -import org.elasticsearch.action.validate.ValidateRequest; -import org.elasticsearch.action.validate.ValidateResponse; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -43,45 +43,45 @@ import static org.elasticsearch.rest.action.support.RestActions.splitTypes; /** * */ -public class RestValidateAction extends BaseRestHandler { +public class RestValidateQueryAction extends BaseRestHandler { @Inject - public RestValidateAction(Settings settings, Client client, RestController controller) { + public RestValidateQueryAction(Settings settings, Client client, RestController controller) { super(settings, client); - controller.registerHandler(GET, "/_validate", this); - controller.registerHandler(POST, "/_validate", this); - controller.registerHandler(GET, "/{index}/_validate", this); - controller.registerHandler(POST, "/{index}/_validate", this); - controller.registerHandler(GET, "/{index}/{type}/_validate", this); - controller.registerHandler(POST, "/{index}/{type}/_validate", this); + controller.registerHandler(GET, "/_validate/query", this); + controller.registerHandler(POST, "/_validate/query", this); + controller.registerHandler(GET, "/{index}/_validate/query", this); + controller.registerHandler(POST, "/{index}/_validate/query", this); + controller.registerHandler(GET, "/{index}/{type}/_validate/query", this); + controller.registerHandler(POST, "/{index}/{type}/_validate/query", this); } @Override public void handleRequest(final RestRequest request, final RestChannel channel) { - ValidateRequest validateRequest = new ValidateRequest(RestActions.splitIndices(request.param("index"))); + ValidateQueryRequest validateQueryRequest = new ValidateQueryRequest(RestActions.splitIndices(request.param("index"))); // we just send back a response, no need to fork a listener - validateRequest.listenerThreaded(false); + validateQueryRequest.listenerThreaded(false); try { BroadcastOperationThreading operationThreading = BroadcastOperationThreading.fromString(request.param("operation_threading"), BroadcastOperationThreading.SINGLE_THREAD); if (operationThreading == BroadcastOperationThreading.NO_THREADS) { // since we don't spawn, don't allow no_threads, but change it to a single thread operationThreading = BroadcastOperationThreading.SINGLE_THREAD; } - validateRequest.operationThreading(operationThreading); + validateQueryRequest.operationThreading(operationThreading); if (request.hasContent()) { - validateRequest.query(request.contentByteArray(), request.contentByteArrayOffset(), request.contentLength(), true); + validateQueryRequest.query(request.contentByteArray(), request.contentByteArrayOffset(), request.contentLength(), true); } else { String source = request.param("source"); if (source != null) { - validateRequest.query(source); + validateQueryRequest.query(source); } else { byte[] querySource = RestActions.parseQuerySource(request); if (querySource != null) { - validateRequest.query(querySource); + validateQueryRequest.query(querySource); } } } - validateRequest.types(splitTypes(request.param("type"))); + validateQueryRequest.types(splitTypes(request.param("type"))); } catch (Exception e) { try { XContentBuilder builder = RestXContentBuilder.restContentBuilder(request); @@ -92,9 +92,9 @@ public class RestValidateAction extends BaseRestHandler { return; } - client.validate(validateRequest, new ActionListener() { + client.admin().indices().validateQuery(validateQueryRequest, new ActionListener() { @Override - public void onResponse(ValidateResponse response) { + public void onResponse(ValidateQueryResponse response) { try { XContentBuilder builder = RestXContentBuilder.restContentBuilder(request); builder.startObject(); diff --git a/src/test/java/org/elasticsearch/test/integration/validate/SimpleValidateTests.java b/src/test/java/org/elasticsearch/test/integration/validate/SimpleValidateQueryTests.java similarity index 68% rename from src/test/java/org/elasticsearch/test/integration/validate/SimpleValidateTests.java rename to src/test/java/org/elasticsearch/test/integration/validate/SimpleValidateQueryTests.java index b117f3abcfb..680374a183f 100644 --- a/src/test/java/org/elasticsearch/test/integration/validate/SimpleValidateTests.java +++ b/src/test/java/org/elasticsearch/test/integration/validate/SimpleValidateQueryTests.java @@ -34,7 +34,7 @@ import static org.hamcrest.Matchers.*; /** * */ -public class SimpleValidateTests extends AbstractNodesTests { +public class SimpleValidateQueryTests extends AbstractNodesTests { private Client client; @@ -70,15 +70,15 @@ public class SimpleValidateTests extends AbstractNodesTests { client.admin().indices().prepareRefresh().execute().actionGet(); - assertThat(client.prepareValidate("test").setQuery("foo".getBytes()).execute().actionGet().valid(), equalTo(false)); - assertThat(client.prepareValidate("test").setQuery(QueryBuilders.queryString("_id:1")).execute().actionGet().valid(), equalTo(true)); - assertThat(client.prepareValidate("test").setQuery(QueryBuilders.queryString("_i:d:1")).execute().actionGet().valid(), equalTo(false)); + assertThat(client.admin().indices().prepareValidateQuery("test").setQuery("foo".getBytes()).execute().actionGet().valid(), equalTo(false)); + assertThat(client.admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryString("_id:1")).execute().actionGet().valid(), equalTo(true)); + assertThat(client.admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryString("_i:d:1")).execute().actionGet().valid(), equalTo(false)); - assertThat(client.prepareValidate("test").setQuery(QueryBuilders.queryString("foo:1")).execute().actionGet().valid(), equalTo(true)); - assertThat(client.prepareValidate("test").setQuery(QueryBuilders.queryString("bar:hey")).execute().actionGet().valid(), equalTo(false)); + assertThat(client.admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryString("foo:1")).execute().actionGet().valid(), equalTo(true)); + assertThat(client.admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryString("bar:hey")).execute().actionGet().valid(), equalTo(false)); - assertThat(client.prepareValidate("test").setQuery(QueryBuilders.queryString("nonexistent:hello")).execute().actionGet().valid(), equalTo(true)); + assertThat(client.admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryString("nonexistent:hello")).execute().actionGet().valid(), equalTo(true)); - assertThat(client.prepareValidate("test").setQuery(QueryBuilders.queryString("foo:1 AND")).execute().actionGet().valid(), equalTo(false)); + assertThat(client.admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryString("foo:1 AND")).execute().actionGet().valid(), equalTo(false)); } } From 73c58693cd5cac0f81778eb48143ca3f3fbb915b Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 29 Dec 2011 13:57:39 +0200 Subject: [PATCH 088/270] improve analyzers and tokenizers bindings to work similar to filters and char filters, by processing them first, and lazily binding them if needed --- .../index/analysis/AnalysisModule.java | 170 ++++++++++++------ 1 file changed, 112 insertions(+), 58 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java b/src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java index 4f1009844b5..7ce840d9cb3 100644 --- a/src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java +++ b/src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java @@ -22,6 +22,7 @@ package org.elasticsearch.index.analysis; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.Scopes; import org.elasticsearch.common.inject.assistedinject.FactoryProvider; @@ -78,25 +79,13 @@ public class AnalysisModule extends AbstractModule { } public static class TokenizersBindings { - private final MapBinder binder; - private final Map groupSettings; - private final IndicesAnalysisService indicesAnalysisService; + private final Map> tokenizers = Maps.newHashMap(); - public TokenizersBindings(MapBinder binder, Map groupSettings, IndicesAnalysisService indicesAnalysisService) { - this.binder = binder; - this.groupSettings = groupSettings; - this.indicesAnalysisService = indicesAnalysisService; + public TokenizersBindings() { } public void processTokenizer(String name, Class tokenizerFactory) { - if (!groupSettings.containsKey(name)) { - if (indicesAnalysisService != null && indicesAnalysisService.hasTokenizer(name)) { - // don't register it here, we will do it in AnalysisService - //binder.addBinding(name).toInstance(indicesAnalysisService.tokenizerFactoryFactory(name)); - } else { - binder.addBinding(name).toProvider(FactoryProvider.newFactory(TokenizerFactoryFactory.class, tokenizerFactory)).in(Scopes.SINGLETON); - } - } + tokenizers.put(name, tokenizerFactory); } } @@ -105,25 +94,13 @@ public class AnalysisModule extends AbstractModule { } public static class AnalyzersBindings { - private final MapBinder binder; - private final Map groupSettings; - private final IndicesAnalysisService indicesAnalysisService; + private final Map> analyzers = Maps.newHashMap(); - public AnalyzersBindings(MapBinder binder, Map groupSettings, IndicesAnalysisService indicesAnalysisService) { - this.binder = binder; - this.groupSettings = groupSettings; - this.indicesAnalysisService = indicesAnalysisService; + public AnalyzersBindings() { } public void processAnalyzer(String name, Class analyzerProvider) { - if (!groupSettings.containsKey(name)) { - if (indicesAnalysisService != null && indicesAnalysisService.hasAnalyzer(name)) { - // don't register here, we will register it in the AnalysisService - //binder.addBinding(name).toInstance(indicesAnalysisService.analyzerProviderFactory(name)); - } else { - binder.addBinding(name).toProvider(FactoryProvider.newFactory(AnalyzerProviderFactory.class, analyzerProvider)).in(Scopes.SINGLETON); - } - } + analyzers.put(name, analyzerProvider); } } } @@ -177,7 +154,13 @@ public class AnalysisModule extends AbstractModule { } catch (NoClassSettingsException e) { // nothing found, see if its in bindings as a binding name if (charFilterSettings.get("type") != null) { - type = charFiltersBindings.charFilters.get(charFilterSettings.get("type")); + type = charFiltersBindings.charFilters.get(Strings.toUnderscoreCase(charFilterSettings.get("type"))); + if (type == null) { + type = charFiltersBindings.charFilters.get(Strings.toCamelCase(charFilterSettings.get("type"))); + } + } + if (type == null) { + throw new ElasticSearchIllegalArgumentException("failed to find char filter type [" + charFilterSettings.get("type") + "] for [" + charFilterName + "]", e); } } if (type == null) { @@ -230,10 +213,13 @@ public class AnalysisModule extends AbstractModule { } catch (NoClassSettingsException e) { // nothing found, see if its in bindings as a binding name if (tokenFilterSettings.get("type") != null) { - type = tokenFiltersBindings.tokenFilters.get(tokenFilterSettings.get("type")); + type = tokenFiltersBindings.tokenFilters.get(Strings.toUnderscoreCase(tokenFilterSettings.get("type"))); + if (type == null) { + type = tokenFiltersBindings.tokenFilters.get(Strings.toCamelCase(tokenFilterSettings.get("type"))); + } } if (type == null) { - throw new ElasticSearchIllegalArgumentException("failed to find token filter type for [" + tokenFilterName + "]", e); + throw new ElasticSearchIllegalArgumentException("failed to find token filter type [" + tokenFilterSettings.get("type") + "] for [" + tokenFilterName + "]", e); } } if (type == null) { @@ -267,31 +253,57 @@ public class AnalysisModule extends AbstractModule { MapBinder tokenizerBinder = MapBinder.newMapBinder(binder(), String.class, TokenizerFactoryFactory.class); + // initial default bindings + AnalysisBinderProcessor.TokenizersBindings tokenizersBindings = new AnalysisBinderProcessor.TokenizersBindings(); + for (AnalysisBinderProcessor processor : processors) { + processor.processTokenizers(tokenizersBindings); + } + Map tokenizersSettings = settings.getGroups("index.analysis.tokenizer"); for (Map.Entry entry : tokenizersSettings.entrySet()) { String tokenizerName = entry.getKey(); Settings tokenizerSettings = entry.getValue(); + + Class type = null; try { - Class type = tokenizerSettings.getAsClass("type", null, "org.elasticsearch.index.analysis.", "TokenizerFactory"); - if (type == null) { - throw new ElasticSearchIllegalArgumentException("Tokenizer [" + tokenizerName + "] must have a type associated with it"); - } - - // if it requires settings, and it has none, then don't register it - if (tokenizerSettings.getAsMap().isEmpty() && type.getAnnotation(AnalysisSettingsRequired.class) != null) { - continue; - } - - tokenizerBinder.addBinding(tokenizerName).toProvider(FactoryProvider.newFactory(TokenizerFactoryFactory.class, type)).in(Scopes.SINGLETON); + type = tokenizerSettings.getAsClass("type", null, "org.elasticsearch.index.analysis.", "TokenizerFactory"); } catch (NoClassSettingsException e) { - throw new ElasticSearchIllegalArgumentException("failed to find tokenizer type for [" + tokenizerName + "]", e); + // nothing found, see if its in bindings as a binding name + if (tokenizerSettings.get("type") != null) { + type = tokenizersBindings.tokenizers.get(Strings.toUnderscoreCase(tokenizerSettings.get("type"))); + if (type == null) { + type = tokenizersBindings.tokenizers.get(Strings.toCamelCase(tokenizerSettings.get("type"))); + } + } + if (type == null) { + throw new ElasticSearchIllegalArgumentException("failed to find tokenizer type [" + tokenizerSettings.get("type") + "] for [" + tokenizerName + "]", e); + } } + if (type == null) { + throw new ElasticSearchIllegalArgumentException("token filter [" + tokenizerName + "] must have a type associated with it"); + } + tokenizerBinder.addBinding(tokenizerName).toProvider(FactoryProvider.newFactory(TokenizerFactoryFactory.class, type)).in(Scopes.SINGLETON); } - - AnalysisBinderProcessor.TokenizersBindings tokenizersBindings = new AnalysisBinderProcessor.TokenizersBindings(tokenizerBinder, tokenizersSettings, indicesAnalysisService); - for (AnalysisBinderProcessor processor : processors) { - processor.processTokenizers(tokenizersBindings); + // go over the tokenizers in the bindings and register the ones that are not configured + for (Map.Entry> entry : tokenizersBindings.tokenizers.entrySet()) { + String tokenizerName = entry.getKey(); + Class clazz = entry.getValue(); + // we don't want to re-register one that already exists + if (tokenizersSettings.containsKey(tokenizerName)) { + continue; + } + // check, if it requires settings, then don't register it, we know default has no settings... + if (clazz.getAnnotation(AnalysisSettingsRequired.class) != null) { + continue; + } + // register it as default under the name + if (indicesAnalysisService != null && indicesAnalysisService.hasTokenizer(tokenizerName)) { + // don't register it here, we will use explicitly register it in the AnalysisService + // tokenizerBinder.addBinding(tokenizerName).toProvider(FactoryProvider.newFactory(TokenizerFactoryFactory.class, clazz)).in(Scopes.SINGLETON); + } else { + tokenizerBinder.addBinding(tokenizerName).toProvider(FactoryProvider.newFactory(TokenizerFactoryFactory.class, clazz)).in(Scopes.SINGLETON); + } } // ANALYZER @@ -299,12 +311,28 @@ public class AnalysisModule extends AbstractModule { MapBinder analyzerBinder = MapBinder.newMapBinder(binder(), String.class, AnalyzerProviderFactory.class); + // initial default bindings + AnalysisBinderProcessor.AnalyzersBindings analyzersBindings = new AnalysisBinderProcessor.AnalyzersBindings(); + for (AnalysisBinderProcessor processor : processors) { + processor.processAnalyzers(analyzersBindings); + } + Map analyzersSettings = settings.getGroups("index.analysis.analyzer"); for (Map.Entry entry : analyzersSettings.entrySet()) { String analyzerName = entry.getKey(); Settings analyzerSettings = entry.getValue(); + + Class type = null; try { - Class type = analyzerSettings.getAsClass("type", null, "org.elasticsearch.index.analysis.", "AnalyzerProvider"); + type = analyzerSettings.getAsClass("type", null, "org.elasticsearch.index.analysis.", "AnalyzerProvider"); + } catch (NoClassSettingsException e) { + // nothing found, see if its in bindings as a binding name + if (analyzerSettings.get("type") != null) { + type = analyzersBindings.analyzers.get(Strings.toUnderscoreCase(analyzerSettings.get("type"))); + if (type == null) { + type = analyzersBindings.analyzers.get(Strings.toCamelCase(analyzerSettings.get("type"))); + } + } if (type == null) { // no specific type, check if it has a tokenizer associated with it String tokenizerName = analyzerSettings.get("tokenizer"); @@ -312,19 +340,45 @@ public class AnalysisModule extends AbstractModule { // we have a tokenizer, use the CustomAnalyzer type = CustomAnalyzerProvider.class; } else { - throw new ElasticSearchIllegalArgumentException("analyzer [" + analyzerName + "] must have a type associated with it or a tokenizer"); + throw new ElasticSearchIllegalArgumentException("failed to find analyzer type [" + analyzerSettings.get("type") + "] or tokenizer for [" + analyzerName + "]", e); } } - analyzerBinder.addBinding(analyzerName).toProvider(FactoryProvider.newFactory(AnalyzerProviderFactory.class, type)).in(Scopes.SINGLETON); - } catch (NoClassSettingsException e) { - throw new ElasticSearchIllegalArgumentException("failed to find analyzer type for [" + analyzerName + "]", e); + } + if (type == null) { + // no specific type, check if it has a tokenizer associated with it + String tokenizerName = analyzerSettings.get("tokenizer"); + if (tokenizerName != null) { + // we have a tokenizer, use the CustomAnalyzer + type = CustomAnalyzerProvider.class; + } else { + throw new ElasticSearchIllegalArgumentException("failed to find analyzer type [" + analyzerSettings.get("type") + "] or tokenizer for [" + analyzerName + "]"); + } + } + analyzerBinder.addBinding(analyzerName).toProvider(FactoryProvider.newFactory(AnalyzerProviderFactory.class, type)).in(Scopes.SINGLETON); + } + + + // go over the tokenizers in the bindings and register the ones that are not configured + for (Map.Entry> entry : analyzersBindings.analyzers.entrySet()) { + String analyzerName = entry.getKey(); + Class clazz = entry.getValue(); + // we don't want to re-register one that already exists + if (analyzersSettings.containsKey(analyzerName)) { + continue; + } + // check, if it requires settings, then don't register it, we know default has no settings... + if (clazz.getAnnotation(AnalysisSettingsRequired.class) != null) { + continue; + } + // register it as default under the name + if (indicesAnalysisService != null && indicesAnalysisService.hasAnalyzer(analyzerName)) { + // don't register it here, we will use explicitly register it in the AnalysisService + // analyzerBinder.addBinding(analyzerName).toProvider(FactoryProvider.newFactory(AnalyzerProviderFactory.class, clazz)).in(Scopes.SINGLETON); + } else { + analyzerBinder.addBinding(analyzerName).toProvider(FactoryProvider.newFactory(AnalyzerProviderFactory.class, clazz)).in(Scopes.SINGLETON); } } - AnalysisBinderProcessor.AnalyzersBindings analyzersBindings = new AnalysisBinderProcessor.AnalyzersBindings(analyzerBinder, analyzersSettings, indicesAnalysisService); - for (AnalysisBinderProcessor processor : processors) { - processor.processAnalyzers(analyzersBindings); - } bind(AnalysisService.class).in(Scopes.SINGLETON); } From c2df2976243b5e4a4a38c3e7808697d8afbb5803 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 29 Dec 2011 14:32:02 +0200 Subject: [PATCH 089/270] randomize routing value when validating query --- .../validate/query/TransportValidateQueryAction.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java index 6ffd48c8275..d148d087e1e 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.indices.validate.query; +import jsr166y.ThreadLocalRandom; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.TransportActions; @@ -34,7 +35,6 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.query.QueryParsingException; -import org.elasticsearch.index.shard.service.IndexShard; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -97,8 +97,8 @@ public class TransportValidateQueryAction extends TransportBroadcastOperationAct @Override protected GroupShardsIterator shards(ValidateQueryRequest request, String[] concreteIndices, ClusterState clusterState) { - // Hard-code routing to limit request to a single shard. - Map> routingMap = clusterState.metaData().resolveSearchRouting("0", request.indices()); + // Hard-code routing to limit request to a single shard, but still, randomize it... + Map> routingMap = clusterState.metaData().resolveSearchRouting(Integer.toString(ThreadLocalRandom.current().nextInt(1000)), request.indices()); return clusterService.operationRouting().searchShards(clusterState, request.indices(), concreteIndices, null, routingMap, "_local"); } From ee011c9f0d9022988b43bce10e050af5ee749119 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 30 Dec 2011 02:00:20 +0200 Subject: [PATCH 090/270] Highlighter: Add require_field_match (both global and per field) option, closes #1577. --- .../search/highlight/HighlightPhase.java | 7 +++++-- .../search/highlight/HighlighterParseElement.java | 11 ++++++++--- .../search/highlight/SearchContextHighlight.java | 9 +++++++++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java b/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java index e07881c0c3f..18134d21858 100644 --- a/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java +++ b/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java @@ -147,7 +147,7 @@ public class HighlightPhase implements FetchSubPhase { } } - QueryScorer queryScorer = new QueryScorer(query, null); + QueryScorer queryScorer = new QueryScorer(query, field.requireFieldMatch() ? mapper.names().indexName() : null); queryScorer.setExpandMultiTermQuery(true); Fragmenter fragmenter; if (field.numberOfFragments() == 0) { @@ -267,10 +267,13 @@ public class HighlightPhase implements FetchSubPhase { entry = new HighlightEntry(); entry.fragListBuilder = fragListBuilder; entry.fragmentsBuilder = fragmentsBuilder; + // parameters to FVH are not requires since: + // first two booleans are not relevant since they are set on the CustomFieldQuery (phrase and fieldMatch) + // fragment builders are used explicitly entry.fvh = new FastVectorHighlighter(true, false, fragListBuilder, fragmentsBuilder); CustomFieldQuery.highlightFilters.set(field.highlightFilter()); // we use top level reader to rewrite the query against all readers, with use caching it across hits (and across readers...) - entry.fieldQuery = new CustomFieldQuery(context.parsedQuery().query(), hitContext.topLevelReader(), entry.fvh); + entry.fieldQuery = new CustomFieldQuery(context.parsedQuery().query(), hitContext.topLevelReader(), true, field.requireFieldMatch()); cache.put(mapper, entry); } diff --git a/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java b/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java index f3aab1dd815..d204ec7b4a7 100644 --- a/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java +++ b/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java @@ -45,8 +45,6 @@ import static com.google.common.collect.Lists.newArrayList; * } * } * - * - * */ public class HighlighterParseElement implements SearchParseElement { @@ -71,6 +69,7 @@ public class HighlighterParseElement implements SearchParseElement { String[] globalPostTags = DEFAULT_POST_TAGS; boolean globalScoreOrdered = false; boolean globalHighlightFilter = false; + boolean globalRequireFieldMatch = false; int globalFragmentSize = 100; int globalNumOfFragments = 5; String globalEncoder = "default"; @@ -109,6 +108,8 @@ public class HighlighterParseElement implements SearchParseElement { globalNumOfFragments = parser.intValue(); } else if ("encoder".equals(topLevelFieldName)) { globalEncoder = parser.text(); + } else if ("require_field_match".equals(topLevelFieldName) || "requireFieldMatch".equals(topLevelFieldName)) { + globalRequireFieldMatch = parser.booleanValue(); } } else if (token == XContentParser.Token.START_OBJECT) { if ("fields".equals(topLevelFieldName)) { @@ -147,6 +148,8 @@ public class HighlighterParseElement implements SearchParseElement { field.highlightFilter(parser.booleanValue()); } else if ("order".equals(fieldName)) { field.scoreOrdered("score".equals(parser.text())); + } else if ("require_field_match".equals(fieldName) || "requireFieldMatch".equals(fieldName)) { + field.requireFieldMatch(parser.booleanValue()); } } } @@ -183,7 +186,9 @@ public class HighlighterParseElement implements SearchParseElement { if (field.encoder() == null) { field.encoder(globalEncoder); } - + if (field.requireFieldMatch() == null) { + field.requireFieldMatch(globalRequireFieldMatch); + } } context.highlight(new SearchContextHighlight(fields)); diff --git a/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java b/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java index 8a58ea281c2..228781b4023 100644 --- a/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java +++ b/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java @@ -56,6 +56,8 @@ public class SearchContextHighlight { private Boolean highlightFilter; + private Boolean requireFieldMatch; + public Field(String field) { this.field = field; } @@ -128,5 +130,12 @@ public class SearchContextHighlight { this.highlightFilter = highlightFilter; } + public Boolean requireFieldMatch() { + return requireFieldMatch; + } + + public void requireFieldMatch(boolean requireFieldMatch) { + this.requireFieldMatch = requireFieldMatch; + } } } From 0f1a908bae7f4703df302a6ee8e1f9348f7584e4 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 30 Dec 2011 02:08:05 +0200 Subject: [PATCH 091/270] improve multi field highlighting with fast vector based highlighting by caching the custom query across field mappers (one when there is field match, and one when there isn't) --- .../search/highlight/HighlightPhase.java | 56 ++++++++++++------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java b/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java index 18134d21858..a7006f79da4 100644 --- a/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java +++ b/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java @@ -84,9 +84,9 @@ public class HighlightPhase implements FetchSubPhase { @Override public void hitExecute(SearchContext context, HitContext hitContext) throws ElasticSearchException { // we use a cache to cache heavy things, mainly the rewrite in FieldQuery for FVH - Map cache = (Map) hitContext.cache().get("highlight"); + HighlighterEntry cache = (HighlighterEntry) hitContext.cache().get("highlight"); if (cache == null) { - cache = Maps.newHashMap(); + cache = new HighlighterEntry(); hitContext.cache().put("highlight", cache); } @@ -119,7 +119,7 @@ public class HighlightPhase implements FetchSubPhase { // if we can do highlighting using Term Vectors, use FastVectorHighlighter, otherwise, use the // slower plain highlighter if (mapper.termVector() != Field.TermVector.WITH_POSITIONS_OFFSETS) { - HighlightEntry entry = cache.get(mapper); + MapperHighlightEntry entry = cache.mappers.get(mapper); if (entry == null) { // Don't use the context.query() since it might be rewritten, and we need to pass the non rewritten queries to // let the highlighter handle MultiTerm ones @@ -158,11 +158,11 @@ public class HighlightPhase implements FetchSubPhase { Formatter formatter = new SimpleHTMLFormatter(field.preTags()[0], field.postTags()[0]); - entry = new HighlightEntry(); + entry = new MapperHighlightEntry(); entry.highlighter = new Highlighter(formatter, encoder, queryScorer); entry.highlighter.setTextFragmenter(fragmenter); - cache.put(mapper, entry); + cache.mappers.put(mapper, entry); } List textsToHighlight; @@ -232,7 +232,8 @@ public class HighlightPhase implements FetchSubPhase { } } else { try { - HighlightEntry entry = cache.get(mapper); + MapperHighlightEntry entry = cache.mappers.get(mapper); + FieldQuery fieldQuery = null; if (entry == null) { FragListBuilder fragListBuilder; FragmentsBuilder fragmentsBuilder; @@ -264,18 +265,30 @@ public class HighlightPhase implements FetchSubPhase { } } } - entry = new HighlightEntry(); + entry = new MapperHighlightEntry(); entry.fragListBuilder = fragListBuilder; entry.fragmentsBuilder = fragmentsBuilder; - // parameters to FVH are not requires since: - // first two booleans are not relevant since they are set on the CustomFieldQuery (phrase and fieldMatch) - // fragment builders are used explicitly - entry.fvh = new FastVectorHighlighter(true, false, fragListBuilder, fragmentsBuilder); + if (cache.fvh == null) { + // parameters to FVH are not requires since: + // first two booleans are not relevant since they are set on the CustomFieldQuery (phrase and fieldMatch) + // fragment builders are used explicitly + cache.fvh = new FastVectorHighlighter(); + } CustomFieldQuery.highlightFilters.set(field.highlightFilter()); - // we use top level reader to rewrite the query against all readers, with use caching it across hits (and across readers...) - entry.fieldQuery = new CustomFieldQuery(context.parsedQuery().query(), hitContext.topLevelReader(), true, field.requireFieldMatch()); - - cache.put(mapper, entry); + if (field.requireFieldMatch()) { + if (cache.fieldMatchFieldQuery == null) { + // we use top level reader to rewrite the query against all readers, with use caching it across hits (and across readers...) + cache.fieldMatchFieldQuery = new CustomFieldQuery(context.parsedQuery().query(), hitContext.topLevelReader(), true, field.requireFieldMatch()); + } + fieldQuery = cache.fieldMatchFieldQuery; + } else { + if (cache.noFieldMatchFieldQuery == null) { + // we use top level reader to rewrite the query against all readers, with use caching it across hits (and across readers...) + cache.noFieldMatchFieldQuery = new CustomFieldQuery(context.parsedQuery().query(), hitContext.topLevelReader(), true, field.requireFieldMatch()); + } + fieldQuery = cache.noFieldMatchFieldQuery; + } + cache.mappers.put(mapper, entry); } String[] fragments; @@ -283,7 +296,7 @@ public class HighlightPhase implements FetchSubPhase { // a HACK to make highlighter do highlighting, even though its using the single frag list builder int numberOfFragments = field.numberOfFragments() == 0 ? 1 : field.numberOfFragments(); // we highlight against the low level reader and docId, because if we load source, we want to reuse it if possible - fragments = entry.fvh.getBestFragments(entry.fieldQuery, hitContext.reader(), hitContext.docId(), mapper.names().indexName(), field.fragmentCharSize(), numberOfFragments, + fragments = cache.fvh.getBestFragments(fieldQuery, hitContext.reader(), hitContext.docId(), mapper.names().indexName(), field.fragmentCharSize(), numberOfFragments, entry.fragListBuilder, entry.fragmentsBuilder, field.preTags(), field.postTags(), encoder); if (fragments != null && fragments.length > 0) { @@ -299,12 +312,17 @@ public class HighlightPhase implements FetchSubPhase { hitContext.hit().highlightFields(highlightFields); } - static class HighlightEntry { - public FastVectorHighlighter fvh; - public FieldQuery fieldQuery; + static class MapperHighlightEntry { public FragListBuilder fragListBuilder; public FragmentsBuilder fragmentsBuilder; public Highlighter highlighter; } + + static class HighlighterEntry { + public FastVectorHighlighter fvh; + public FieldQuery noFieldMatchFieldQuery; + public FieldQuery fieldMatchFieldQuery; + public Map mappers = Maps.newHashMap(); + } } From 3c88eacb01b53085b83b0533a4f5d9a5425e2b3f Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 30 Dec 2011 13:42:08 +0200 Subject: [PATCH 092/270] bind transport validate query actions as singletons --- .../java/org/elasticsearch/action/TransportActionModule.java | 2 ++ .../client/transport/action/ClientTransportActionModule.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/main/java/org/elasticsearch/action/TransportActionModule.java b/src/main/java/org/elasticsearch/action/TransportActionModule.java index 8b19f9d6fe3..2b2c5119f8e 100644 --- a/src/main/java/org/elasticsearch/action/TransportActionModule.java +++ b/src/main/java/org/elasticsearch/action/TransportActionModule.java @@ -52,6 +52,7 @@ import org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction; import org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction; import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.put.TransportPutIndexTemplateAction; +import org.elasticsearch.action.admin.indices.validate.query.TransportValidateQueryAction; import org.elasticsearch.action.bulk.TransportBulkAction; import org.elasticsearch.action.bulk.TransportShardBulkAction; import org.elasticsearch.action.count.TransportCountAction; @@ -110,6 +111,7 @@ public class TransportActionModule extends AbstractModule { bind(TransportAnalyzeAction.class).asEagerSingleton(); bind(TransportPutIndexTemplateAction.class).asEagerSingleton(); bind(TransportDeleteIndexTemplateAction.class).asEagerSingleton(); + bind(TransportValidateQueryAction.class).asEagerSingleton(); bind(TransportGatewaySnapshotAction.class).asEagerSingleton(); diff --git a/src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java b/src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java index 6bd0d45ad6d..a8b7fdac8ee 100644 --- a/src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java +++ b/src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java @@ -50,6 +50,7 @@ import org.elasticsearch.client.transport.action.admin.indices.stats.ClientTrans import org.elasticsearch.client.transport.action.admin.indices.status.ClientTransportIndicesStatusAction; import org.elasticsearch.client.transport.action.admin.indices.template.delete.ClientTransportDeleteIndexTemplateAction; import org.elasticsearch.client.transport.action.admin.indices.template.put.ClientTransportPutIndexTemplateAction; +import org.elasticsearch.client.transport.action.admin.indices.validate.query.ClientTransportValidateQueryAction; import org.elasticsearch.client.transport.action.bulk.ClientTransportBulkAction; import org.elasticsearch.client.transport.action.count.ClientTransportCountAction; import org.elasticsearch.client.transport.action.delete.ClientTransportDeleteAction; @@ -100,6 +101,7 @@ public class ClientTransportActionModule extends AbstractModule { bind(ClientTransportAnalyzeAction.class).asEagerSingleton(); bind(ClientTransportPutIndexTemplateAction.class).asEagerSingleton(); bind(ClientTransportDeleteIndexTemplateAction.class).asEagerSingleton(); + bind(ClientTransportValidateQueryAction.class).asEagerSingleton(); bind(ClientTransportNodesInfoAction.class).asEagerSingleton(); bind(ClientTransportNodesStatsAction.class).asEagerSingleton(); From a4e4235d93a12cebe397f5225ebc2de1a174dda8 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 31 Dec 2011 16:24:27 +0200 Subject: [PATCH 093/270] move sort optimization when sorting by _score to sort parse element from the query phase execution --- .../elasticsearch/search/query/QueryPhase.java | 16 +--------------- .../search/sort/SortParseElement.java | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 21dffcb6001..a305d73f2e2 100644 --- a/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -176,20 +176,6 @@ public class QueryPhase implements SearchPhase { // if 0 was asked, change it to 1 since 0 is not allowed numDocs = 1; } - boolean sort = false; - // try and optimize for a case where the sorting is based on score, this is how we work by default! - if (searchContext.sort() != null) { - if (searchContext.sort().getSort().length > 1) { - sort = true; - } else { - SortField sortField = searchContext.sort().getSort()[0]; - if (sortField.getType() == SortField.SCORE && !sortField.getReverse()) { - sort = false; - } else { - sort = true; - } - } - } if (searchContext.searchType() == SearchType.COUNT) { CountCollector countCollector = new CountCollector(); @@ -207,7 +193,7 @@ public class QueryPhase implements SearchPhase { // all is well } topDocs = scanCollector.topDocs(); - } else if (sort) { + } else if (searchContext.sort() != null) { topDocs = searchContext.searcher().search(query, null, numDocs, searchContext.sort()); } else { topDocs = searchContext.searcher().search(query, numDocs); diff --git a/src/main/java/org/elasticsearch/search/sort/SortParseElement.java b/src/main/java/org/elasticsearch/search/sort/SortParseElement.java index bbf2a51b9df..eb72e40c023 100644 --- a/src/main/java/org/elasticsearch/search/sort/SortParseElement.java +++ b/src/main/java/org/elasticsearch/search/sort/SortParseElement.java @@ -76,7 +76,21 @@ public class SortParseElement implements SearchParseElement { addCompoundSortField(parser, context, sortFields); } if (!sortFields.isEmpty()) { - context.sort(new Sort(sortFields.toArray(new SortField[sortFields.size()]))); + // optimize if we just sort on score non reversed, we don't really need sorting + boolean sort; + if (sortFields.size() > 1) { + sort = true; + } else { + SortField sortField = sortFields.get(0); + if (sortField.getType() == SortField.SCORE && !sortField.getReverse()) { + sort = false; + } else { + sort = true; + } + } + if (sort) { + context.sort(new Sort(sortFields.toArray(new SortField[sortFields.size()]))); + } } } From 370444cb75ea1017e166a34cf7f719c6428f1ad6 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 31 Dec 2011 16:26:01 +0200 Subject: [PATCH 094/270] count collector does not throw stop collection --- .../java/org/elasticsearch/search/query/QueryPhase.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/src/main/java/org/elasticsearch/search/query/QueryPhase.java index a305d73f2e2..8d3244e8d37 100644 --- a/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -179,11 +179,7 @@ public class QueryPhase implements SearchPhase { if (searchContext.searchType() == SearchType.COUNT) { CountCollector countCollector = new CountCollector(); - try { - searchContext.searcher().search(query, countCollector); - } catch (ScanCollector.StopCollectingException e) { - // all is well - } + searchContext.searcher().search(query, countCollector); topDocs = countCollector.topDocs(); } else if (searchContext.searchType() == SearchType.SCAN) { ScanCollector scanCollector = new ScanCollector(searchContext.from(), searchContext.size(), searchContext.trackScores()); From 8cf8b478af17e680c6ae1eb2d4546b2b8a21c284 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 31 Dec 2011 17:49:19 +0200 Subject: [PATCH 095/270] Scan Search: Improve performance while scrolling through it, closes #1579. --- .../elasticsearch/common/lucene/Lucene.java | 2 + .../search/internal/SearchContext.java | 14 ++ .../search/query/QueryPhase.java | 110 +------------ .../search/scan/ScanContext.java | 147 ++++++++++++++++++ .../search/scan/SearchScanScrollingTests.java | 4 + .../search/scan/SearchScanTests.java | 8 + 6 files changed, 180 insertions(+), 105 deletions(-) create mode 100644 src/main/java/org/elasticsearch/search/scan/ScanContext.java diff --git a/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/src/main/java/org/elasticsearch/common/lucene/Lucene.java index 22321e3647a..e1620f5d1d2 100644 --- a/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -53,6 +53,8 @@ public class Lucene { public static final int NO_DOC = -1; + public static ScoreDoc[] EMPTY_SCORE_DOCS = new ScoreDoc[0]; + public static Version parseVersion(@Nullable String version, Version defaultVersion, ESLogger logger) { if (version == null) { return defaultVersion; diff --git a/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 36847015558..68e644542d0 100644 --- a/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -55,6 +55,7 @@ import org.elasticsearch.search.fetch.script.ScriptFieldsContext; import org.elasticsearch.search.highlight.SearchContextHighlight; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.query.QuerySearchResult; +import org.elasticsearch.search.scan.ScanContext; import java.util.ArrayList; import java.util.HashMap; @@ -106,6 +107,9 @@ public class SearchContext implements Releasable { private final FetchSearchResult fetchResult; + // lazy initialized only if needed + private ScanContext scanContext; + private float queryBoost = 1.0f; @@ -180,6 +184,9 @@ public class SearchContext implements Releasable { @Override public boolean release() throws ElasticSearchException { + if (scanContext != null) { + scanContext.clear(); + } // clear and scope phase we have if (scopePhases != null) { for (ScopePhase scopePhase : scopePhases) { @@ -547,6 +554,13 @@ public class SearchContext implements Releasable { nestedQueries.put(scope, query); } + public ScanContext scanContext() { + if (scanContext == null) { + scanContext = new ScanContext(); + } + return scanContext; + } + public MapperService.SmartNameFieldMappers smartFieldMappers(String name) { return mapperService().smartName(name, request.types()); } diff --git a/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 8d3244e8d37..5311551ca64 100644 --- a/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -20,10 +20,10 @@ package org.elasticsearch.search.query; import com.google.common.collect.ImmutableMap; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.*; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.function.BoostScoreFunction; import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; import org.elasticsearch.index.query.ParsedQuery; @@ -36,8 +36,6 @@ import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.sort.SortParseElement; import org.elasticsearch.search.sort.TrackScoresParseElement; -import java.io.IOException; -import java.util.ArrayList; import java.util.Map; /** @@ -178,17 +176,11 @@ public class QueryPhase implements SearchPhase { } if (searchContext.searchType() == SearchType.COUNT) { - CountCollector countCollector = new CountCollector(); - searchContext.searcher().search(query, countCollector); - topDocs = countCollector.topDocs(); + TotalHitCountCollector collector = new TotalHitCountCollector(); + searchContext.searcher().search(query, collector); + topDocs = new TopDocs(collector.getTotalHits(), Lucene.EMPTY_SCORE_DOCS, 0); } else if (searchContext.searchType() == SearchType.SCAN) { - ScanCollector scanCollector = new ScanCollector(searchContext.from(), searchContext.size(), searchContext.trackScores()); - try { - searchContext.searcher().search(query, scanCollector); - } catch (ScanCollector.StopCollectingException e) { - // all is well - } - topDocs = scanCollector.topDocs(); + topDocs = searchContext.scanContext().execute(searchContext); } else if (searchContext.sort() != null) { topDocs = searchContext.searcher().search(query, null, numDocs, searchContext.sort()); } else { @@ -203,96 +195,4 @@ public class QueryPhase implements SearchPhase { facetPhase.execute(searchContext); } - - static class CountCollector extends Collector { - - private int totalHits = 0; - - @Override - public void setScorer(Scorer scorer) throws IOException { - } - - @Override - public void collect(int doc) throws IOException { - totalHits++; - } - - @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { - } - - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - - public TopDocs topDocs() { - return new TopDocs(totalHits, EMPTY, 0); - } - - private static ScoreDoc[] EMPTY = new ScoreDoc[0]; - } - - static class ScanCollector extends Collector { - - private final int from; - - private final int to; - - private final ArrayList docs; - - private final boolean trackScores; - - private Scorer scorer; - - private int docBase; - - private int counter; - - ScanCollector(int from, int size, boolean trackScores) { - this.from = from; - this.to = from + size; - this.trackScores = trackScores; - this.docs = new ArrayList(size); - } - - public TopDocs topDocs() { - return new TopDocs(docs.size(), docs.toArray(new ScoreDoc[docs.size()]), 0f); - } - - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - } - - @Override - public void collect(int doc) throws IOException { - if (counter >= from) { - docs.add(new ScoreDoc(docBase + doc, trackScores ? scorer.score() : 0f)); - } - counter++; - if (counter >= to) { - throw StopCollectingException; - } - } - - @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { - this.docBase = docBase; - } - - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - - public static final RuntimeException StopCollectingException = new StopCollectingException(); - - static class StopCollectingException extends RuntimeException { - @Override - public Throwable fillInStackTrace() { - return null; - } - } - } } diff --git a/src/main/java/org/elasticsearch/search/scan/ScanContext.java b/src/main/java/org/elasticsearch/search/scan/ScanContext.java new file mode 100644 index 00000000000..3f5590d2de2 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/scan/ScanContext.java @@ -0,0 +1,147 @@ +package org.elasticsearch.search.scan; + +import com.google.common.collect.Maps; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.*; +import org.elasticsearch.common.lucene.docset.AllDocSet; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Map; + +/** + * The scan context allows to optimize readers we already processed during scanning. We do that by keeping track + * of the count per reader, and if we are done with it, we no longer process it by using a filter that returns + * null docIdSet for this reader. + */ +public class ScanContext { + + private final Map readerStates = Maps.newHashMap(); + + public void clear() { + readerStates.clear(); + } + + public TopDocs execute(SearchContext context) throws IOException { + ScanCollector collector = new ScanCollector(readerStates, context.from(), context.size(), context.trackScores()); + Query query = new FilteredQuery(context.query(), new ScanFilter(readerStates, collector)); + try { + context.searcher().search(query, collector); + } catch (ScanCollector.StopCollectingException e) { + // all is well + } + return collector.topDocs(); + } + + static class ScanCollector extends Collector { + + private final Map readerStates; + + private final int from; + + private final int to; + + private final ArrayList docs; + + private final boolean trackScores; + + private Scorer scorer; + + private int docBase; + + private int counter; + + private IndexReader currentReader; + private ReaderState readerState; + + ScanCollector(Map readerStates, int from, int size, boolean trackScores) { + this.readerStates = readerStates; + this.from = from; + this.to = from + size; + this.trackScores = trackScores; + this.docs = new ArrayList(size); + } + + void incCounter(int count) { + this.counter += count; + } + + public TopDocs topDocs() { + return new TopDocs(docs.size(), docs.toArray(new ScoreDoc[docs.size()]), 0f); + } + + @Override + public void setScorer(Scorer scorer) throws IOException { + this.scorer = scorer; + } + + @Override + public void collect(int doc) throws IOException { + if (counter >= from) { + docs.add(new ScoreDoc(docBase + doc, trackScores ? scorer.score() : 0f)); + } + readerState.count++; + counter++; + if (counter >= to) { + throw StopCollectingException; + } + } + + @Override + public void setNextReader(IndexReader reader, int docBase) throws IOException { + // if we have a reader state, and we haven't registered one already, register it + // we need to check in readersState since even when the filter return null, setNextReader is still + // called for that reader (before) + if (currentReader != null && !readerStates.containsKey(currentReader)) { + assert readerState != null; + readerState.done = true; + readerStates.put(currentReader, readerState); + } + this.currentReader = reader; + this.docBase = docBase; + this.readerState = new ReaderState(); + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return true; + } + + public static final RuntimeException StopCollectingException = new StopCollectingException(); + + static class StopCollectingException extends RuntimeException { + @Override + public Throwable fillInStackTrace() { + return null; + } + } + } + + public static class ScanFilter extends Filter { + + private final Map readerStates; + + private final ScanCollector scanCollector; + + public ScanFilter(Map readerStates, ScanCollector scanCollector) { + this.readerStates = readerStates; + this.scanCollector = scanCollector; + } + + @Override + public DocIdSet getDocIdSet(IndexReader reader) throws IOException { + ReaderState readerState = readerStates.get(reader); + if (readerState != null && readerState.done) { + scanCollector.incCounter(readerState.count); + return null; + } + return new AllDocSet(reader.maxDoc()); + } + } + + static class ReaderState { + public int count; + public boolean done; + } +} diff --git a/src/test/java/org/elasticsearch/test/integration/search/scan/SearchScanScrollingTests.java b/src/test/java/org/elasticsearch/test/integration/search/scan/SearchScanScrollingTests.java index 024376bffd1..5d18b006d09 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/scan/SearchScanScrollingTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/scan/SearchScanScrollingTests.java @@ -222,6 +222,10 @@ public class SearchScanScrollingTests extends AbstractNodesTests { } } client.prepareIndex("test", "type1", id).setRouting(routing).setSource("field", i).execute().actionGet(); + // make some segments + if (i % 10 == 0) { + client.admin().indices().prepareFlush().execute().actionGet(); + } } client.admin().indices().prepareRefresh().execute().actionGet(); diff --git a/src/test/java/org/elasticsearch/test/integration/search/scan/SearchScanTests.java b/src/test/java/org/elasticsearch/test/integration/search/scan/SearchScanTests.java index eee377a157b..434d4112420 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/scan/SearchScanTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/scan/SearchScanTests.java @@ -77,11 +77,19 @@ public class SearchScanTests extends AbstractNodesTests { expectedIds.add(Integer.toString(i)); client.prepareIndex("test", "tweet", Integer.toString(i)).setSource( jsonBuilder().startObject().field("user", "kimchy1").field("postDate", System.currentTimeMillis()).field("message", "test").endObject()).execute().actionGet(); + // make some segments + if (i % 10 == 0) { + client.admin().indices().prepareFlush().execute().actionGet(); + } } for (int i = 100; i < 200; i++) { client.prepareIndex("test", "tweet", Integer.toString(i)).setSource( jsonBuilder().startObject().field("user", "kimchy2").field("postDate", System.currentTimeMillis()).field("message", "test").endObject()).execute().actionGet(); + // make some segments + if (i % 10 == 0) { + client.admin().indices().prepareFlush().execute().actionGet(); + } } client.admin().indices().prepareRefresh().execute().actionGet(); From 53b1b6e4bb94e1db1bbca764100f2e0206b43f4d Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 31 Dec 2011 20:33:17 +0200 Subject: [PATCH 096/270] add size in bytes to bb dir --- .../lucene/store/bytebuffer/ByteBufferDirectory.java | 12 ++++++++++++ .../lucene/store/bytebuffer/ByteBufferFile.java | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferDirectory.java b/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferDirectory.java index 657f2e1c363..3d56531ed99 100644 --- a/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferDirectory.java +++ b/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferDirectory.java @@ -28,6 +28,7 @@ import java.nio.ByteBuffer; import java.util.Collection; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; /** * A memory based directory that uses {@link java.nio.ByteBuffer} in order to store the directory content. @@ -48,6 +49,8 @@ public class ByteBufferDirectory extends Directory { private final boolean internalAllocator; + final AtomicLong sizeInBytes = new AtomicLong(); + /** * Constructs a new directory using {@link PlainByteBufferAllocator}. */ @@ -74,6 +77,13 @@ public class ByteBufferDirectory extends Directory { } } + /** + * Returns the size in bytes of the directory, chunk by buffer size. + */ + public long sizeInBytes() { + return sizeInBytes.get(); + } + public void sync(Collection names) throws IOException { // nothing to do here } @@ -123,6 +133,7 @@ public class ByteBufferDirectory extends Directory { ByteBufferFile file = files.remove(name); if (file == null) throw new FileNotFoundException(name); + sizeInBytes.addAndGet(-file.sizeInBytes.get()); file.delete(); } @@ -143,6 +154,7 @@ public class ByteBufferDirectory extends Directory { ByteBufferFile file = new ByteBufferFile(this, allocator.sizeInBytes(allocatorType)); ByteBufferFile existing = files.put(name, file); if (existing != null) { + sizeInBytes.addAndGet(-existing.sizeInBytes.get()); existing.delete(); } return new ByteBufferIndexOutput(name, allocator, allocatorType, file); diff --git a/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferFile.java b/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferFile.java index 92627caa2c7..4ec37074137 100644 --- a/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferFile.java +++ b/src/main/java/org/apache/lucene/store/bytebuffer/ByteBufferFile.java @@ -20,6 +20,7 @@ package org.apache.lucene.store.bytebuffer; import java.nio.ByteBuffer; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; /** */ @@ -35,6 +36,8 @@ public class ByteBufferFile { private final AtomicInteger refCount = new AtomicInteger(1); + final AtomicLong sizeInBytes = new AtomicLong(); + public ByteBufferFile(ByteBufferDirectory dir, int bufferSize) { this.dir = dir; this.bufferSize = bufferSize; @@ -60,6 +63,8 @@ public class ByteBufferFile { protected final void addBuffer(ByteBuffer buffer) { buffers.add(buffer); + sizeInBytes.addAndGet(buffer.remaining()); + dir.sizeInBytes.addAndGet(buffer.remaining()); } protected final ByteBuffer getBuffer(int index) { From 8c6b2a30773a805aeaf9668f6496a93581326f45 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 1 Jan 2012 00:09:57 +0200 Subject: [PATCH 097/270] Date Histogram Facet: Improve time zone handling, add factor option, closes #1580. --- .../common/joda/TimeZoneRounding.java | 240 ++++++++++++++++++ .../CountDateHistogramFacetCollector.java | 49 +--- .../DateHistogramFacetProcessor.java | 118 ++++----- .../ValueDateHistogramFacetCollector.java | 28 +- ...alueScriptDateHistogramFacetCollector.java | 60 +---- .../search/facet/SimpleFacetsTests.java | 12 +- .../unit/deps/joda/TimeZoneRoundingTests.java | 85 +++++++ 7 files changed, 411 insertions(+), 181 deletions(-) create mode 100644 src/main/java/org/elasticsearch/common/joda/TimeZoneRounding.java create mode 100644 src/test/java/org/elasticsearch/test/unit/deps/joda/TimeZoneRoundingTests.java diff --git a/src/main/java/org/elasticsearch/common/joda/TimeZoneRounding.java b/src/main/java/org/elasticsearch/common/joda/TimeZoneRounding.java new file mode 100644 index 00000000000..ac4cbe3d2f0 --- /dev/null +++ b/src/main/java/org/elasticsearch/common/joda/TimeZoneRounding.java @@ -0,0 +1,240 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.joda; + +import org.elasticsearch.common.unit.TimeValue; +import org.joda.time.DateTimeConstants; +import org.joda.time.DateTimeField; +import org.joda.time.DateTimeZone; + +/** + */ +public abstract class TimeZoneRounding { + + public abstract long calc(long utcMillis); + + public static Builder builder(DateTimeField field) { + return new Builder(field); + } + + public static Builder builder(TimeValue interval) { + return new Builder(interval); + } + + public static class Builder { + + private DateTimeField field; + private long interval = -1; + + private DateTimeZone preTz = DateTimeZone.UTC; + private DateTimeZone postTz = DateTimeZone.UTC; + + private float factor = 1.0f; + + public Builder(DateTimeField field) { + this.field = field; + this.interval = -1; + } + + public Builder(TimeValue interval) { + this.field = null; + this.interval = interval.millis(); + } + + public Builder preZone(DateTimeZone preTz) { + this.preTz = preTz; + return this; + } + + public Builder postZone(DateTimeZone postTz) { + this.postTz = postTz; + return this; + } + + public Builder factor(float factor) { + this.factor = factor; + return this; + } + + public TimeZoneRounding build() { + TimeZoneRounding timeZoneRounding; + if (field != null) { + if (preTz.equals(DateTimeZone.UTC) && postTz.equals(DateTimeZone.UTC)) { + return new UTCTimeZoneRoundingFloor(field); + } else if (field.getDurationField().getUnitMillis() < DateTimeConstants.MILLIS_PER_HOUR * 12) { + timeZoneRounding = new TimeTimeZoneRoundingFloor(field, preTz, postTz); + } else { + timeZoneRounding = new DayTimeZoneRoundingFloor(field, preTz, postTz); + } + } else { + if (preTz.equals(DateTimeZone.UTC) && postTz.equals(DateTimeZone.UTC)) { + return new UTCIntervalTimeZoneRounding(interval); + } else if (interval < DateTimeConstants.MILLIS_PER_HOUR * 12) { + timeZoneRounding = new TimeIntervalTimeZoneRounding(interval, preTz, postTz); + } else { + timeZoneRounding = new DayIntervalTimeZoneRounding(interval, preTz, postTz); + } + } + if (factor != 1.0f) { + timeZoneRounding = new FactorTimeZoneRounding(timeZoneRounding, factor); + } + return timeZoneRounding; + } + } + + static class TimeTimeZoneRoundingFloor extends TimeZoneRounding { + + private final DateTimeField field; + private final DateTimeZone preTz; + private final DateTimeZone postTz; + + TimeTimeZoneRoundingFloor(DateTimeField field, DateTimeZone preTz, DateTimeZone postTz) { + this.field = field; + this.preTz = preTz; + this.postTz = postTz; + } + + @Override + public long calc(long utcMillis) { + long time = utcMillis + preTz.getOffset(utcMillis); + time = field.roundFloor(time); + // now, time is still in local, move it to UTC + time = time - preTz.getOffset(time); + // now apply post Tz + time = time + postTz.getOffset(time); + return time; + } + } + + static class UTCTimeZoneRoundingFloor extends TimeZoneRounding { + + private final DateTimeField field; + + UTCTimeZoneRoundingFloor(DateTimeField field) { + this.field = field; + } + + @Override + public long calc(long utcMillis) { + return field.roundFloor(utcMillis); + } + } + + static class DayTimeZoneRoundingFloor extends TimeZoneRounding { + private final DateTimeField field; + private final DateTimeZone preTz; + private final DateTimeZone postTz; + + DayTimeZoneRoundingFloor(DateTimeField field, DateTimeZone preTz, DateTimeZone postTz) { + this.field = field; + this.preTz = preTz; + this.postTz = postTz; + } + + @Override + public long calc(long utcMillis) { + long time = utcMillis + preTz.getOffset(utcMillis); + time = field.roundFloor(time); + // after rounding, since its day level (and above), its actually UTC! + // now apply post Tz + time = time + postTz.getOffset(time); + return time; + } + } + + static class UTCIntervalTimeZoneRounding extends TimeZoneRounding { + + private final long interval; + + UTCIntervalTimeZoneRounding(long interval) { + this.interval = interval; + } + + @Override + public long calc(long utcMillis) { + return ((utcMillis / interval) * interval); + } + } + + + static class TimeIntervalTimeZoneRounding extends TimeZoneRounding { + + private final long interval; + private final DateTimeZone preTz; + private final DateTimeZone postTz; + + TimeIntervalTimeZoneRounding(long interval, DateTimeZone preTz, DateTimeZone postTz) { + this.interval = interval; + this.preTz = preTz; + this.postTz = postTz; + } + + @Override + public long calc(long utcMillis) { + long time = utcMillis + preTz.getOffset(utcMillis); + time = ((time / interval) * interval); + // now, time is still in local, move it to UTC + time = time - preTz.getOffset(time); + // now apply post Tz + time = time + postTz.getOffset(time); + return time; + } + } + + static class DayIntervalTimeZoneRounding extends TimeZoneRounding { + + private final long interval; + private final DateTimeZone preTz; + private final DateTimeZone postTz; + + DayIntervalTimeZoneRounding(long interval, DateTimeZone preTz, DateTimeZone postTz) { + this.interval = interval; + this.preTz = preTz; + this.postTz = postTz; + } + + @Override + public long calc(long utcMillis) { + long time = utcMillis + preTz.getOffset(utcMillis); + time = ((time / interval) * interval); + // after rounding, since its day level (and above), its actually UTC! + // now apply post Tz + time = time + postTz.getOffset(time); + return time; + } + } + + static class FactorTimeZoneRounding extends TimeZoneRounding { + + private final TimeZoneRounding timeZoneRounding; + + private final float factor; + + FactorTimeZoneRounding(TimeZoneRounding timeZoneRounding, float factor) { + this.timeZoneRounding = timeZoneRounding; + this.factor = factor; + } + + @Override + public long calc(long utcMillis) { + return timeZoneRounding.calc((long) (factor * utcMillis)); + } + } +} diff --git a/src/main/java/org/elasticsearch/search/facet/datehistogram/CountDateHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/datehistogram/CountDateHistogramFacetCollector.java index e34ba452552..7f610acf00a 100644 --- a/src/main/java/org/elasticsearch/search/facet/datehistogram/CountDateHistogramFacetCollector.java +++ b/src/main/java/org/elasticsearch/search/facet/datehistogram/CountDateHistogramFacetCollector.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.facet.datehistogram; import gnu.trove.map.hash.TLongLongHashMap; import org.apache.lucene.index.IndexReader; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.joda.TimeZoneRounding; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; import org.elasticsearch.index.field.data.longs.LongFieldData; @@ -31,22 +32,17 @@ import org.elasticsearch.search.facet.AbstractFacetCollector; import org.elasticsearch.search.facet.Facet; import org.elasticsearch.search.facet.FacetPhaseExecutionException; import org.elasticsearch.search.internal.SearchContext; -import org.joda.time.MutableDateTime; import java.io.IOException; /** * A date histogram facet collector that uses the same field as the key as well as the * value. - * - * */ public class CountDateHistogramFacetCollector extends AbstractFacetCollector { private final String indexFieldName; - private final MutableDateTime dateTime; - private final DateHistogramFacet.ComparatorType comparatorType; private final FieldDataCache fieldDataCache; @@ -57,9 +53,8 @@ public class CountDateHistogramFacetCollector extends AbstractFacetCollector { private final DateHistogramProc histoProc; - public CountDateHistogramFacetCollector(String facetName, String fieldName, MutableDateTime dateTime, long interval, DateHistogramFacet.ComparatorType comparatorType, SearchContext context) { + public CountDateHistogramFacetCollector(String facetName, String fieldName, TimeZoneRounding tzRounding, DateHistogramFacet.ComparatorType comparatorType, SearchContext context) { super(facetName); - this.dateTime = dateTime; this.comparatorType = comparatorType; this.fieldDataCache = context.fieldDataCache(); @@ -77,17 +72,12 @@ public class CountDateHistogramFacetCollector extends AbstractFacetCollector { indexFieldName = mapper.names().indexName(); fieldDataType = mapper.fieldDataType(); - - if (interval == 1) { - histoProc = new DateHistogramProc(); - } else { - histoProc = new IntervalDateHistogramProc(interval); - } + histoProc = new DateHistogramProc(tzRounding); } @Override protected void doCollect(int doc) throws IOException { - fieldData.forEachValueInDoc(doc, dateTime, histoProc); + fieldData.forEachValueInDoc(doc, histoProc); } @Override @@ -100,36 +90,23 @@ public class CountDateHistogramFacetCollector extends AbstractFacetCollector { return new InternalCountDateHistogramFacet(facetName, comparatorType, histoProc.counts(), true); } - public static long bucket(long value, long interval) { - return ((value / interval) * interval); - } + public static class DateHistogramProc implements LongFieldData.LongValueInDocProc { - public static class DateHistogramProc implements LongFieldData.DateValueInDocProc { + private final TLongLongHashMap counts = CacheRecycler.popLongLongMap(); - protected final TLongLongHashMap counts = CacheRecycler.popLongLongMap(); + private final TimeZoneRounding tzRounding; + + public DateHistogramProc(TimeZoneRounding tzRounding) { + this.tzRounding = tzRounding; + } @Override - public void onValue(int docId, MutableDateTime dateTime) { - counts.adjustOrPutValue(dateTime.getMillis(), 1, 1); + public void onValue(int docId, long value) { + counts.adjustOrPutValue(tzRounding.calc(value), 1, 1); } public TLongLongHashMap counts() { return counts; } } - - public static class IntervalDateHistogramProc extends DateHistogramProc { - - private final long interval; - - public IntervalDateHistogramProc(long interval) { - this.interval = interval; - } - - @Override - public void onValue(int docId, MutableDateTime dateTime) { - long bucket = bucket(dateTime.getMillis(), interval); - counts.adjustOrPutValue(bucket, 1, 1); - } - } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java b/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java index 73199af739d..54199862da0 100644 --- a/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java +++ b/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java @@ -20,11 +20,10 @@ package org.elasticsearch.search.facet.datehistogram; import com.google.common.collect.ImmutableMap; -import gnu.trove.impl.Constants; -import gnu.trove.map.hash.TObjectIntHashMap; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.joda.TimeZoneRounding; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; @@ -38,7 +37,7 @@ import org.elasticsearch.search.internal.SearchContext; import org.joda.time.Chronology; import org.joda.time.DateTimeField; import org.joda.time.DateTimeZone; -import org.joda.time.MutableDateTime; +import org.joda.time.chrono.ISOChronology; import java.io.IOException; import java.util.List; @@ -50,7 +49,6 @@ import java.util.Map; public class DateHistogramFacetProcessor extends AbstractComponent implements FacetProcessor { private final ImmutableMap dateFieldParsers; - private final TObjectIntHashMap rounding = new TObjectIntHashMap(Constants.DEFAULT_CAPACITY, Constants.DEFAULT_LOAD_FACTOR, -1); @Inject public DateHistogramFacetProcessor(Settings settings) { @@ -73,15 +71,6 @@ public class DateHistogramFacetProcessor extends AbstractComponent implements Fa .put("second", new DateFieldParser.SecondOfMinute()) .put("1s", new DateFieldParser.SecondOfMinute()) .immutableMap(); - - rounding.put("floor", MutableDateTime.ROUND_FLOOR); - rounding.put("ceiling", MutableDateTime.ROUND_CEILING); - rounding.put("half_even", MutableDateTime.ROUND_HALF_EVEN); - rounding.put("halfEven", MutableDateTime.ROUND_HALF_EVEN); - rounding.put("half_floor", MutableDateTime.ROUND_HALF_FLOOR); - rounding.put("halfFloor", MutableDateTime.ROUND_HALF_FLOOR); - rounding.put("half_ceiling", MutableDateTime.ROUND_HALF_CEILING); - rounding.put("halfCeiling", MutableDateTime.ROUND_HALF_CEILING); } @Override @@ -96,10 +85,11 @@ public class DateHistogramFacetProcessor extends AbstractComponent implements Fa String valueScript = null; String scriptLang = null; Map params = null; - boolean intervalSet = false; - long interval = 1; - String sInterval = null; - MutableDateTime dateTime = new MutableDateTime(DateTimeZone.UTC); + String interval = null; + DateTimeZone preZone = DateTimeZone.UTC; + DateTimeZone postZone = DateTimeZone.UTC; + float factor = 1.0f; + Chronology chronology = ISOChronology.getInstanceUTC(); DateHistogramFacet.ComparatorType comparatorType = DateHistogramFacet.ComparatorType.TIME; XContentParser.Token token; String fieldName = null; @@ -118,29 +108,15 @@ public class DateHistogramFacetProcessor extends AbstractComponent implements Fa } else if ("value_field".equals(fieldName) || "valueField".equals(fieldName)) { valueField = parser.text(); } else if ("interval".equals(fieldName)) { - intervalSet = true; - if (token == XContentParser.Token.VALUE_NUMBER) { - interval = parser.longValue(); - } else { - sInterval = parser.text(); - } + interval = parser.text(); } else if ("time_zone".equals(fieldName) || "timeZone".equals(fieldName)) { - if (token == XContentParser.Token.VALUE_NUMBER) { - dateTime.setZone(DateTimeZone.forOffsetHours(parser.intValue())); - } else { - String text = parser.text(); - int index = text.indexOf(':'); - if (index != -1) { - // format like -02:30 - dateTime.setZone(DateTimeZone.forOffsetHoursMinutes( - Integer.parseInt(text.substring(0, index)), - Integer.parseInt(text.substring(index + 1)) - )); - } else { - // id, listed here: http://joda-time.sourceforge.net/timezones.html - dateTime.setZone(DateTimeZone.forID(text)); - } - } + preZone = parseZone(parser, token); + } else if ("pre_zone".equals(fieldName) || "preZone".equals(fieldName)) { + preZone = parseZone(parser, token); + } else if ("post_zone".equals(fieldName) || "postZone".equals(fieldName)) { + postZone = parseZone(parser, token); + } else if ("factor".equals(fieldName)) { + factor = parser.floatValue(); } else if ("value_script".equals(fieldName) || "valueScript".equals(fieldName)) { valueScript = parser.text(); } else if ("order".equals(fieldName) || "comparator".equals(fieldName)) { @@ -163,48 +139,46 @@ public class DateHistogramFacetProcessor extends AbstractComponent implements Fa throw new FacetPhaseExecutionException(facetName, "(key) field [" + keyField + "] is not of type date"); } - if (!intervalSet) { + if (interval == null) { throw new FacetPhaseExecutionException(facetName, "[interval] is required to be set for histogram facet"); } - // we set the rounding after we set the zone, for it to take affect - if (sInterval != null) { - int index = sInterval.indexOf(':'); - if (index != -1) { - // set with rounding - DateFieldParser fieldParser = dateFieldParsers.get(sInterval.substring(0, index)); - if (fieldParser == null) { - throw new FacetPhaseExecutionException(facetName, "failed to parse interval [" + sInterval + "] with custom rounding using built in intervals (year/month/...)"); - } - DateTimeField field = fieldParser.parse(dateTime.getChronology()); - int rounding = this.rounding.get(sInterval.substring(index + 1)); - if (rounding == -1) { - throw new FacetPhaseExecutionException(facetName, "failed to parse interval [" + sInterval + "], rounding type [" + (sInterval.substring(index + 1)) + "] not found"); - } - dateTime.setRounding(field, rounding); - } else { - DateFieldParser fieldParser = dateFieldParsers.get(sInterval); - if (fieldParser != null) { - DateTimeField field = fieldParser.parse(dateTime.getChronology()); - dateTime.setRounding(field, MutableDateTime.ROUND_FLOOR); - } else { - // time interval - try { - interval = TimeValue.parseTimeValue(sInterval, null).millis(); - } catch (Exception e) { - throw new FacetPhaseExecutionException(facetName, "failed to parse interval [" + sInterval + "], tried both as built in intervals (year/month/...) and as a time format"); - } - } - } + TimeZoneRounding.Builder tzRoundingBuilder; + DateFieldParser fieldParser = dateFieldParsers.get(interval); + if (fieldParser != null) { + tzRoundingBuilder = TimeZoneRounding.builder(fieldParser.parse(chronology)); + } else { + // the interval is a time value? + tzRoundingBuilder = TimeZoneRounding.builder(TimeValue.parseTimeValue(interval, null)); } + TimeZoneRounding tzRounding = tzRoundingBuilder.preZone(preZone).postZone(postZone).factor(factor).build(); if (valueScript != null) { - return new ValueScriptDateHistogramFacetCollector(facetName, keyField, scriptLang, valueScript, params, dateTime, interval, comparatorType, context); + return new ValueScriptDateHistogramFacetCollector(facetName, keyField, scriptLang, valueScript, params, tzRounding, comparatorType, context); } else if (valueField == null) { - return new CountDateHistogramFacetCollector(facetName, keyField, dateTime, interval, comparatorType, context); + return new CountDateHistogramFacetCollector(facetName, keyField, tzRounding, comparatorType, context); } else { - return new ValueDateHistogramFacetCollector(facetName, keyField, valueField, dateTime, interval, comparatorType, context); + return new ValueDateHistogramFacetCollector(facetName, keyField, valueField, tzRounding, comparatorType, context); + } + } + + private DateTimeZone parseZone(XContentParser parser, XContentParser.Token token) throws IOException { + if (token == XContentParser.Token.VALUE_NUMBER) { + return DateTimeZone.forOffsetHours(parser.intValue()); + } else { + String text = parser.text(); + int index = text.indexOf(':'); + if (index != -1) { + // format like -02:30 + return DateTimeZone.forOffsetHoursMinutes( + Integer.parseInt(text.substring(0, index)), + Integer.parseInt(text.substring(index + 1)) + ); + } else { + // id, listed here: http://joda-time.sourceforge.net/timezones.html + return DateTimeZone.forID(text); + } } } diff --git a/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueDateHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueDateHistogramFacetCollector.java index a29baad629d..e17e5a307f7 100644 --- a/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueDateHistogramFacetCollector.java +++ b/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueDateHistogramFacetCollector.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.facet.datehistogram; import org.apache.lucene.index.IndexReader; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.joda.TimeZoneRounding; import org.elasticsearch.common.trove.ExtTLongObjectHashMap; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; @@ -32,7 +33,6 @@ import org.elasticsearch.search.facet.AbstractFacetCollector; import org.elasticsearch.search.facet.Facet; import org.elasticsearch.search.facet.FacetPhaseExecutionException; import org.elasticsearch.search.internal.SearchContext; -import org.joda.time.MutableDateTime; import java.io.IOException; @@ -44,9 +44,6 @@ public class ValueDateHistogramFacetCollector extends AbstractFacetCollector { private final String keyIndexFieldName; private final String valueIndexFieldName; - private MutableDateTime dateTime; - private final long interval; - private final DateHistogramFacet.ComparatorType comparatorType; private final FieldDataCache fieldDataCache; @@ -58,10 +55,8 @@ public class ValueDateHistogramFacetCollector extends AbstractFacetCollector { private final DateHistogramProc histoProc; - public ValueDateHistogramFacetCollector(String facetName, String keyFieldName, String valueFieldName, MutableDateTime dateTime, long interval, DateHistogramFacet.ComparatorType comparatorType, SearchContext context) { + public ValueDateHistogramFacetCollector(String facetName, String keyFieldName, String valueFieldName, TimeZoneRounding tzRounding, DateHistogramFacet.ComparatorType comparatorType, SearchContext context) { super(facetName); - this.dateTime = dateTime; - this.interval = interval; this.comparatorType = comparatorType; this.fieldDataCache = context.fieldDataCache(); @@ -85,12 +80,12 @@ public class ValueDateHistogramFacetCollector extends AbstractFacetCollector { valueIndexFieldName = mapper.names().indexName(); valueFieldDataType = mapper.fieldDataType(); - this.histoProc = new DateHistogramProc(interval); + this.histoProc = new DateHistogramProc(tzRounding); } @Override protected void doCollect(int doc) throws IOException { - keyFieldData.forEachValueInDoc(doc, dateTime, histoProc); + keyFieldData.forEachValueInDoc(doc, histoProc); } @Override @@ -104,26 +99,23 @@ public class ValueDateHistogramFacetCollector extends AbstractFacetCollector { return new InternalFullDateHistogramFacet(facetName, comparatorType, histoProc.entries, true); } - public static class DateHistogramProc implements LongFieldData.DateValueInDocProc { + public static class DateHistogramProc implements LongFieldData.LongValueInDocProc { final ExtTLongObjectHashMap entries = CacheRecycler.popLongObjectMap(); - private final long interval; + private final TimeZoneRounding tzRounding; NumericFieldData valueFieldData; final ValueAggregator valueAggregator = new ValueAggregator(); - public DateHistogramProc(long interval) { - this.interval = interval; + public DateHistogramProc(TimeZoneRounding tzRounding) { + this.tzRounding = tzRounding; } @Override - public void onValue(int docId, MutableDateTime dateTime) { - long time = dateTime.getMillis(); - if (interval != 1) { - time = CountDateHistogramFacetCollector.bucket(time, interval); - } + public void onValue(int docId, long value) { + long time = tzRounding.calc(value); InternalFullDateHistogramFacet.FullEntry entry = entries.get(time); if (entry == null) { diff --git a/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java index cd0ce4515bd..58c1e9d4346 100644 --- a/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java +++ b/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.facet.datehistogram; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Scorer; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.joda.TimeZoneRounding; import org.elasticsearch.common.trove.ExtTLongObjectHashMap; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; @@ -33,7 +34,6 @@ import org.elasticsearch.search.facet.AbstractFacetCollector; import org.elasticsearch.search.facet.Facet; import org.elasticsearch.search.facet.FacetPhaseExecutionException; import org.elasticsearch.search.internal.SearchContext; -import org.joda.time.MutableDateTime; import java.io.IOException; import java.util.Map; @@ -46,8 +46,6 @@ public class ValueScriptDateHistogramFacetCollector extends AbstractFacetCollect private final String indexFieldName; - private final MutableDateTime dateTime; - private final DateHistogramFacet.ComparatorType comparatorType; private final FieldDataCache fieldDataCache; @@ -60,9 +58,8 @@ public class ValueScriptDateHistogramFacetCollector extends AbstractFacetCollect private final DateHistogramProc histoProc; - public ValueScriptDateHistogramFacetCollector(String facetName, String fieldName, String scriptLang, String valueScript, Map params, MutableDateTime dateTime, long interval, DateHistogramFacet.ComparatorType comparatorType, SearchContext context) { + public ValueScriptDateHistogramFacetCollector(String facetName, String fieldName, String scriptLang, String valueScript, Map params, TimeZoneRounding tzRounding, DateHistogramFacet.ComparatorType comparatorType, SearchContext context) { super(facetName); - this.dateTime = dateTime; this.comparatorType = comparatorType; this.fieldDataCache = context.fieldDataCache(); @@ -83,16 +80,12 @@ public class ValueScriptDateHistogramFacetCollector extends AbstractFacetCollect indexFieldName = mapper.names().indexName(); fieldDataType = mapper.fieldDataType(); - if (interval == 1) { - histoProc = new DateHistogramProc(this.valueScript); - } else { - histoProc = new IntervalDateHistogramProc(interval, this.valueScript); - } + histoProc = new DateHistogramProc(tzRounding, this.valueScript); } @Override protected void doCollect(int doc) throws IOException { - fieldData.forEachValueInDoc(doc, dateTime, histoProc); + fieldData.forEachValueInDoc(doc, histoProc); } @Override @@ -111,54 +104,23 @@ public class ValueScriptDateHistogramFacetCollector extends AbstractFacetCollect return new InternalFullDateHistogramFacet(facetName, comparatorType, histoProc.entries, true); } - public static class DateHistogramProc implements LongFieldData.DateValueInDocProc { + public static class DateHistogramProc implements LongFieldData.LongValueInDocProc { + + private final TimeZoneRounding tzRounding; protected final SearchScript valueScript; final ExtTLongObjectHashMap entries = CacheRecycler.popLongObjectMap(); - public DateHistogramProc(SearchScript valueScript) { + public DateHistogramProc(TimeZoneRounding tzRounding, SearchScript valueScript) { + this.tzRounding = tzRounding; this.valueScript = valueScript; } @Override - public void onValue(int docId, MutableDateTime dateTime) { + public void onValue(int docId, long value) { valueScript.setNextDocId(docId); - long time = dateTime.getMillis(); - double scriptValue = valueScript.runAsDouble(); - - InternalFullDateHistogramFacet.FullEntry entry = entries.get(time); - if (entry == null) { - entry = new InternalFullDateHistogramFacet.FullEntry(time, 1, scriptValue, scriptValue, 1, scriptValue); - entries.put(time, entry); - } else { - entry.count++; - entry.totalCount++; - entry.total += scriptValue; - if (scriptValue < entry.min) { - entry.min = scriptValue; - } - if (scriptValue > entry.max) { - entry.max = scriptValue; - } - } - } - } - - public static class IntervalDateHistogramProc extends DateHistogramProc { - - private final long interval; - - public IntervalDateHistogramProc(long interval, SearchScript valueScript) { - super(valueScript); - this.interval = interval; - } - - @Override - public void onValue(int docId, MutableDateTime dateTime) { - valueScript.setNextDocId(docId); - - long time = CountDateHistogramFacetCollector.bucket(dateTime.getMillis(), interval); + long time = tzRounding.calc(value); double scriptValue = valueScript.runAsDouble(); InternalFullDateHistogramFacet.FullEntry entry = entries.get(time); diff --git a/src/test/java/org/elasticsearch/test/integration/search/facet/SimpleFacetsTests.java b/src/test/java/org/elasticsearch/test/integration/search/facet/SimpleFacetsTests.java index c2a89ab490d..a0d9649d00c 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/facet/SimpleFacetsTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/facet/SimpleFacetsTests.java @@ -1424,19 +1424,19 @@ public class SimpleFacetsTests extends AbstractNodesTests { facet = searchResponse.facets().facet("stats2"); assertThat(facet.name(), equalTo("stats2")); assertThat(facet.entries().size(), equalTo(2)); - assertThat(facet.entries().get(0).time(), equalTo(timeInMillis("2009-03-04", DateTimeZone.forOffsetHours(-2)))); + assertThat(facet.entries().get(0).time(), equalTo(utcTimeInMillis("2009-03-04"))); assertThat(facet.entries().get(0).count(), equalTo(1l)); - assertThat(facet.entries().get(1).time(), equalTo(timeInMillis("2009-03-05", DateTimeZone.forOffsetHours(-2)))); + assertThat(facet.entries().get(1).time(), equalTo(utcTimeInMillis("2009-03-05"))); assertThat(facet.entries().get(1).count(), equalTo(2l)); // time zone causes the dates to shift by 2 facet = searchResponse.facets().facet("stats3"); assertThat(facet.name(), equalTo("stats3")); assertThat(facet.entries().size(), equalTo(2)); - assertThat(facet.entries().get(0).time(), equalTo(timeInMillis("2009-03-04", DateTimeZone.forOffsetHours(-2)))); + assertThat(facet.entries().get(0).time(), equalTo(utcTimeInMillis("2009-03-04"))); assertThat(facet.entries().get(0).count(), equalTo(1l)); assertThat(facet.entries().get(0).total(), equalTo(1d)); - assertThat(facet.entries().get(1).time(), equalTo(timeInMillis("2009-03-05", DateTimeZone.forOffsetHours(-2)))); + assertThat(facet.entries().get(1).time(), equalTo(utcTimeInMillis("2009-03-05"))); assertThat(facet.entries().get(1).count(), equalTo(2l)); assertThat(facet.entries().get(1).total(), equalTo(5d)); @@ -1444,10 +1444,10 @@ public class SimpleFacetsTests extends AbstractNodesTests { facet = searchResponse.facets().facet("stats4"); assertThat(facet.name(), equalTo("stats4")); assertThat(facet.entries().size(), equalTo(2)); - assertThat(facet.entries().get(0).time(), equalTo(timeInMillis("2009-03-04", DateTimeZone.forOffsetHours(-2)))); + assertThat(facet.entries().get(0).time(), equalTo(utcTimeInMillis("2009-03-04"))); assertThat(facet.entries().get(0).count(), equalTo(1l)); assertThat(facet.entries().get(0).total(), equalTo(2d)); - assertThat(facet.entries().get(1).time(), equalTo(timeInMillis("2009-03-05", DateTimeZone.forOffsetHours(-2)))); + assertThat(facet.entries().get(1).time(), equalTo(utcTimeInMillis("2009-03-05"))); assertThat(facet.entries().get(1).count(), equalTo(2l)); assertThat(facet.entries().get(1).total(), equalTo(10d)); diff --git a/src/test/java/org/elasticsearch/test/unit/deps/joda/TimeZoneRoundingTests.java b/src/test/java/org/elasticsearch/test/unit/deps/joda/TimeZoneRoundingTests.java new file mode 100644 index 00000000000..9d59516f1b5 --- /dev/null +++ b/src/test/java/org/elasticsearch/test/unit/deps/joda/TimeZoneRoundingTests.java @@ -0,0 +1,85 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.unit.deps.joda; + +import org.elasticsearch.common.joda.TimeZoneRounding; +import org.elasticsearch.common.unit.TimeValue; +import org.joda.time.Chronology; +import org.joda.time.DateTimeZone; +import org.joda.time.chrono.ISOChronology; +import org.joda.time.format.ISODateTimeFormat; +import org.testng.annotations.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +/** + */ +@Test +public class TimeZoneRoundingTests { + + @Test + public void testUTCMonthRounding() { + TimeZoneRounding tzRounding = TimeZoneRounding.builder(chronology().monthOfYear()).build(); + assertThat(tzRounding.calc(utc("2009-02-03T01:01:01")), equalTo(utc("2009-02-01T00:00:00.000Z"))); + } + + @Test + public void testDayTimeZoneRounding() { + TimeZoneRounding tzRounding = TimeZoneRounding.builder(chronology().dayOfMonth()).preZone(DateTimeZone.forOffsetHours(-2)).build(); + assertThat(tzRounding.calc(0), equalTo(0l - TimeValue.timeValueHours(24).millis())); + + tzRounding = TimeZoneRounding.builder(chronology().dayOfMonth()).preZone(DateTimeZone.forOffsetHours(-2)).postZone(DateTimeZone.forOffsetHours(-2)).build(); + assertThat(tzRounding.calc(0), equalTo(0l - TimeValue.timeValueHours(26).millis())); + + tzRounding = TimeZoneRounding.builder(chronology().dayOfMonth()).preZone(DateTimeZone.forOffsetHours(-2)).build(); + assertThat(tzRounding.calc(utc("2009-02-03T01:01:01")), equalTo(utc("2009-02-02T00:00:00"))); + + tzRounding = TimeZoneRounding.builder(chronology().dayOfMonth()).preZone(DateTimeZone.forOffsetHours(-2)).postZone(DateTimeZone.forOffsetHours(-2)).build(); + assertThat(tzRounding.calc(utc("2009-02-03T01:01:01")), equalTo(time("2009-02-02T00:00:00", DateTimeZone.forOffsetHours(+2)))); + } + + @Test + public void testTimeTimeZoneRounding() { + TimeZoneRounding tzRounding = TimeZoneRounding.builder(chronology().hourOfDay()).preZone(DateTimeZone.forOffsetHours(-2)).build(); + assertThat(tzRounding.calc(0), equalTo(0l)); + + tzRounding = TimeZoneRounding.builder(chronology().hourOfDay()).preZone(DateTimeZone.forOffsetHours(-2)).postZone(DateTimeZone.forOffsetHours(-2)).build(); + assertThat(tzRounding.calc(0), equalTo(0l - TimeValue.timeValueHours(2).millis())); + + tzRounding = TimeZoneRounding.builder(chronology().hourOfDay()).preZone(DateTimeZone.forOffsetHours(-2)).build(); + assertThat(tzRounding.calc(utc("2009-02-03T01:01:01")), equalTo(utc("2009-02-03T01:00:00"))); + + tzRounding = TimeZoneRounding.builder(chronology().hourOfDay()).preZone(DateTimeZone.forOffsetHours(-2)).postZone(DateTimeZone.forOffsetHours(-2)).build(); + assertThat(tzRounding.calc(utc("2009-02-03T01:01:01")), equalTo(time("2009-02-03T01:00:00", DateTimeZone.forOffsetHours(+2)))); + } + + private static Chronology chronology() { + return ISOChronology.getInstanceUTC(); + } + + private long utc(String time) { + return time(time, DateTimeZone.UTC); + } + + private long time(String time, DateTimeZone zone) { + return ISODateTimeFormat.dateOptionalTimeParser().withZone(zone).parseMillis(time); + } +} From e582f6c91ab98faf0fd4322247a63b320c3a1419 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Sat, 31 Dec 2011 17:55:44 -0700 Subject: [PATCH 098/270] Rename count/pacakge-info.java to count/package-info.java --- .../action/count/{pacakge-info.java => package-info.java} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/java/org/elasticsearch/action/count/{pacakge-info.java => package-info.java} (100%) diff --git a/src/main/java/org/elasticsearch/action/count/pacakge-info.java b/src/main/java/org/elasticsearch/action/count/package-info.java similarity index 100% rename from src/main/java/org/elasticsearch/action/count/pacakge-info.java rename to src/main/java/org/elasticsearch/action/count/package-info.java From 83d5084f620e13678e7786b5c735de32c4e9fb80 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 2 Jan 2012 22:02:19 +0200 Subject: [PATCH 099/270] Update API: Allow to update a document based on a script, closes #1583. --- .../action/TransportActionModule.java | 2 + .../action/TransportActions.java | 2 + .../InstanceShardOperationRequest.java | 108 ++++++ ...ransportInstanceSingleOperationAction.java | 306 +++++++++++++++++ .../action/update/TransportUpdateAction.java | 254 ++++++++++++++ .../action/update/UpdateRequest.java | 325 ++++++++++++++++++ .../action/update/UpdateResponse.java | 122 +++++++ .../java/org/elasticsearch/client/Client.java | 30 +- .../action/update/UpdateRequestBuilder.java | 157 +++++++++ .../elasticsearch/client/node/NodeClient.java | 18 +- .../client/support/AbstractClient.java | 11 + .../client/transport/TransportClient.java | 14 +- .../action/ClientTransportActionModule.java | 2 + .../update/ClientTransportUpdateAction.java | 44 +++ .../support/InternalTransportClient.java | 28 +- .../cluster/metadata/MappingMetaData.java | 2 +- .../common/io/stream/StreamInput.java | 64 ++++ .../common/io/stream/StreamOutput.java | 68 ++++ .../common/xcontent/XContentHelper.java | 17 +- .../common/xcontent/XContentParser.java | 2 + .../support/AbstractXContentParser.java | 9 + .../DocumentMIssingEngineException.java | 38 ++ .../DocumentSourceMissingEngineException.java | 38 ++ .../elasticsearch/index/get/GetResult.java | 7 + .../mapper/internal/SourceFieldMapper.java | 2 +- .../rest/action/RestActionModule.java | 11 +- .../rest/action/update/RestUpdateAction.java | 146 ++++++++ .../search/lookup/SourceLookup.java | 2 +- .../DiscoveryTransportClientTests.yml | 6 - .../TransportClientDocumentActionsTests.java | 9 + .../TransportClientDocumentActionsTests.yml | 12 - ...ransportClientMoreLikeThisActionTests.java | 56 --- ...TransportClientMoreLikeThisActionTests.yml | 6 - ...nsportClientSniffDocumentActionsTests.java | 9 + ...ansportClientSniffDocumentActionsTests.yml | 12 - .../AliasedIndexDocumentActionsTests.java | 8 + .../AliasedIndexDocumentActionsTests.yml | 8 - .../document/DocumentActionsTests.java | 63 +++- .../document/DocumentActionsTests.yml | 6 - .../document/LocalDocumentActionsTests.java | 8 + .../document/LocalDocumentActionsTests.yml | 8 - .../document/MoreLikeThisActionTests.yml | 6 - .../{document => get}/GetActionTests.java | 2 +- .../MoreLikeThisActionTests.java | 2 +- 44 files changed, 1906 insertions(+), 144 deletions(-) create mode 100644 src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java create mode 100644 src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java create mode 100644 src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java create mode 100644 src/main/java/org/elasticsearch/action/update/UpdateRequest.java create mode 100644 src/main/java/org/elasticsearch/action/update/UpdateResponse.java create mode 100644 src/main/java/org/elasticsearch/client/action/update/UpdateRequestBuilder.java create mode 100644 src/main/java/org/elasticsearch/client/transport/action/update/ClientTransportUpdateAction.java create mode 100644 src/main/java/org/elasticsearch/index/engine/DocumentMIssingEngineException.java create mode 100644 src/main/java/org/elasticsearch/index/engine/DocumentSourceMissingEngineException.java create mode 100644 src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java delete mode 100644 src/test/java/org/elasticsearch/test/integration/client/transport/DiscoveryTransportClientTests.yml delete mode 100644 src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientDocumentActionsTests.yml delete mode 100644 src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientMoreLikeThisActionTests.java delete mode 100644 src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientMoreLikeThisActionTests.yml delete mode 100644 src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientSniffDocumentActionsTests.yml delete mode 100644 src/test/java/org/elasticsearch/test/integration/document/AliasedIndexDocumentActionsTests.yml delete mode 100644 src/test/java/org/elasticsearch/test/integration/document/DocumentActionsTests.yml delete mode 100644 src/test/java/org/elasticsearch/test/integration/document/LocalDocumentActionsTests.yml delete mode 100644 src/test/java/org/elasticsearch/test/integration/document/MoreLikeThisActionTests.yml rename src/test/java/org/elasticsearch/test/integration/{document => get}/GetActionTests.java (99%) rename src/test/java/org/elasticsearch/test/integration/{document => mlt}/MoreLikeThisActionTests.java (99%) diff --git a/src/main/java/org/elasticsearch/action/TransportActionModule.java b/src/main/java/org/elasticsearch/action/TransportActionModule.java index 2b2c5119f8e..334f1d0911a 100644 --- a/src/main/java/org/elasticsearch/action/TransportActionModule.java +++ b/src/main/java/org/elasticsearch/action/TransportActionModule.java @@ -71,6 +71,7 @@ import org.elasticsearch.action.percolate.TransportPercolateAction; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.search.TransportSearchScrollAction; import org.elasticsearch.action.search.type.*; +import org.elasticsearch.action.update.TransportUpdateAction; import org.elasticsearch.common.inject.AbstractModule; /** @@ -126,6 +127,7 @@ public class TransportActionModule extends AbstractModule { bind(TransportIndexDeleteAction.class).asEagerSingleton(); bind(TransportShardDeleteAction.class).asEagerSingleton(); bind(TransportCountAction.class).asEagerSingleton(); + bind(TransportUpdateAction.class).asEagerSingleton(); bind(TransportMultiGetAction.class).asEagerSingleton(); bind(TransportShardMultiGetAction.class).asEagerSingleton(); diff --git a/src/main/java/org/elasticsearch/action/TransportActions.java b/src/main/java/org/elasticsearch/action/TransportActions.java index a0c7ab46851..9e6c061e518 100644 --- a/src/main/java/org/elasticsearch/action/TransportActions.java +++ b/src/main/java/org/elasticsearch/action/TransportActions.java @@ -28,6 +28,8 @@ public class TransportActions { public static final String INDEX = "indices/index/shard/index"; + public static final String UPDATE = "update"; + public static final String COUNT = "indices/count"; public static final String DELETE = "indices/index/shard/delete"; diff --git a/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java b/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java new file mode 100644 index 00000000000..5e1a3941eb8 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java @@ -0,0 +1,108 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.support.single.instance; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.Actions; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.TimeValue; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +/** + * + */ +public abstract class InstanceShardOperationRequest implements ActionRequest { + + public static final TimeValue DEFAULT_TIMEOUT = new TimeValue(1, TimeUnit.MINUTES); + + protected TimeValue timeout = DEFAULT_TIMEOUT; + + protected String index; + // -1 means its not set, allows to explicitly direct a request to a specific shard + protected int shardId = -1; + + private boolean threadedListener = false; + + protected InstanceShardOperationRequest() { + } + + public InstanceShardOperationRequest(String index) { + this.index = index; + } + + public TimeValue timeout() { + return timeout; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (index == null) { + validationException = Actions.addValidationError("index is missing", validationException); + } + return validationException; + } + + public String index() { + return index; + } + + InstanceShardOperationRequest index(String index) { + this.index = index; + return this; + } + + /** + * Should the listener be called on a separate thread if needed. + */ + @Override + public boolean listenerThreaded() { + return threadedListener; + } + + @Override + public InstanceShardOperationRequest listenerThreaded(boolean threadedListener) { + this.threadedListener = threadedListener; + return this; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + index = in.readUTF(); + shardId = in.readInt(); + timeout = TimeValue.readTimeValue(in); + // no need to pass threading over the network, they are always false when coming throw a thread pool + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeUTF(index); + out.writeInt(shardId); + timeout.writeTo(out); + } + + public void beforeLocalFork() { + } +} + diff --git a/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java new file mode 100644 index 00000000000..406935e719c --- /dev/null +++ b/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java @@ -0,0 +1,306 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.support.single.instance; + +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.UnavailableShardsException; +import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.TimeoutClusterStateListener; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.ShardIterator; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.node.NodeClosedException; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.*; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * + */ +public abstract class TransportInstanceSingleOperationAction extends BaseAction { + + protected final ClusterService clusterService; + + protected final TransportService transportService; + + final String transportAction; + final String executor; + + protected TransportInstanceSingleOperationAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService) { + super(settings, threadPool); + this.clusterService = clusterService; + this.transportService = transportService; + + this.transportAction = transportAction(); + this.executor = executor(); + + transportService.registerHandler(transportAction, new TransportHandler()); + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + new AsyncSingleAction(request, listener).start(); + } + + protected abstract String executor(); + + protected abstract String transportAction(); + + protected abstract void shardOperation(Request request, ActionListener listener) throws ElasticSearchException; + + protected abstract Request newRequest(); + + protected abstract Response newResponse(); + + protected void checkBlock(Request request, ClusterState state) { + + } + + protected boolean retryRequired(ShardIterator shardIt, ClusterState state) { + return false; + } + + protected boolean retryOnFailure(Throwable e) { + return false; + } + + protected TransportRequestOptions transportOptions() { + return TransportRequestOptions.EMPTY; + } + + /** + * Should return an iterator with a single shard! + */ + protected abstract ShardIterator shards(ClusterState clusterState, Request request) throws ElasticSearchException; + + class AsyncSingleAction { + + private final ActionListener listener; + + private final Request request; + + private ShardIterator shardIt; + + private DiscoveryNodes nodes; + + private final AtomicBoolean operationStarted = new AtomicBoolean(); + + private AsyncSingleAction(Request request, ActionListener listener) { + this.request = request; + this.listener = listener; + + ClusterState clusterState = clusterService.state(); + + nodes = clusterState.nodes(); + + request.index(clusterState.metaData().concreteIndex(request.index())); + + checkBlock(request, clusterState); + } + + public void start() { + start(false); + } + + public boolean start(final boolean fromClusterEvent) throws ElasticSearchException { + final ClusterState clusterState = clusterService.state(); + nodes = clusterState.nodes(); + if (!clusterState.routingTable().hasIndex(request.index())) { + retry(fromClusterEvent); + return false; + } + try { + shardIt = shards(clusterState, request); + } catch (Exception e) { + listener.onFailure(e); + return true; + } + + // no shardIt, might be in the case between index gateway recovery and shardIt initialization + if (shardIt.size() == 0) { + retry(fromClusterEvent); + return false; + } + + // this transport only make sense with an iterator that returns a single shard routing (like primary) + assert shardIt.size() == 1; + + ShardRouting shard = shardIt.nextOrNull(); + assert shard != null; + + if (!shard.active()) { + retry(fromClusterEvent); + return false; + } + + if (!operationStarted.compareAndSet(false, true)) { + return true; + } + + request.shardId = shardIt.shardId().id(); + if (shard.currentNodeId().equals(nodes.localNodeId())) { + request.beforeLocalFork(); + threadPool.executor(executor).execute(new Runnable() { + @Override + public void run() { + try { + shardOperation(request, listener); + } catch (Exception e) { + if (retryOnFailure(e)) { + retry(fromClusterEvent); + } else { + listener.onFailure(e); + } + } + } + }); + } else { + DiscoveryNode node = nodes.get(shard.currentNodeId()); + transportService.sendRequest(node, transportAction, request, transportOptions(), new BaseTransportResponseHandler() { + + @Override + public Response newInstance() { + return newResponse(); + } + + @Override + public String executor() { + return ThreadPool.Names.SAME; + } + + @Override + public void handleResponse(Response response) { + listener.onResponse(response); + } + + @Override + public void handleException(TransportException exp) { + // if we got disconnected from the node, or the node / shard is not in the right state (being closed) + if (exp.unwrapCause() instanceof ConnectTransportException || exp.unwrapCause() instanceof NodeClosedException || + retryOnFailure(exp)) { + operationStarted.set(false); + // we already marked it as started when we executed it (removed the listener) so pass false + // to re-add to the cluster listener + retry(false); + } else { + listener.onFailure(exp); + } + } + }); + } + return true; + } + + void retry(boolean fromClusterEvent) { + if (!fromClusterEvent) { + // make it threaded operation so we fork on the discovery listener thread + request.beforeLocalFork(); + clusterService.add(request.timeout(), new TimeoutClusterStateListener() { + @Override + public void postAdded() { + if (start(true)) { + // if we managed to start and perform the operation on the primary, we can remove this listener + clusterService.remove(this); + } + } + + @Override + public void onClose() { + clusterService.remove(this); + listener.onFailure(new NodeClosedException(nodes.localNode())); + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (start(true)) { + // if we managed to start and perform the operation on the primary, we can remove this listener + clusterService.remove(this); + } + } + + @Override + public void onTimeout(TimeValue timeValue) { + // just to be on the safe side, see if we can start it now? + if (start(true)) { + clusterService.remove(this); + return; + } + clusterService.remove(this); + final UnavailableShardsException failure; + if (shardIt == null) { + failure = new UnavailableShardsException(new ShardId(request.index(), -1), "Timeout waiting for [" + timeValue + "], request: " + request.toString()); + } else { + failure = new UnavailableShardsException(shardIt.shardId(), "[" + shardIt.size() + "] shardIt, [" + shardIt.sizeActive() + "] active : Timeout waiting for [" + timeValue + "], request: " + request.toString()); + } + listener.onFailure(failure); + } + }); + } + } + } + + class TransportHandler extends BaseTransportRequestHandler { + + @Override + public Request newInstance() { + return newRequest(); + } + + @Override + public String executor() { + return ThreadPool.Names.SAME; + } + + @Override + public void messageReceived(Request request, final TransportChannel channel) throws Exception { + // no need to have a threaded listener since we just send back a response + request.listenerThreaded(false); + execute(request, new ActionListener() { + @Override + public void onResponse(Response result) { + try { + channel.sendResponse(result); + } catch (Exception e) { + onFailure(e); + } + } + + @Override + public void onFailure(Throwable e) { + try { + channel.sendResponse(e); + } catch (Exception e1) { + logger.warn("Failed to send response for get", e1); + } + } + }); + } + } +} diff --git a/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java new file mode 100644 index 00000000000..7234317afe4 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -0,0 +1,254 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.update; + +import com.google.common.collect.ImmutableList; +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.TransportActions; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.delete.TransportDeleteAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.index.TransportIndexAction; +import org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction; +import org.elasticsearch.client.Requests; +import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.routing.PlainShardIterator; +import org.elasticsearch.cluster.routing.ShardIterator; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.engine.DocumentMissingEngineException; +import org.elasticsearch.index.engine.DocumentSourceMissingEngineException; +import org.elasticsearch.index.engine.VersionConflictEngineException; +import org.elasticsearch.index.get.GetResult; +import org.elasticsearch.index.mapper.internal.ParentFieldMapper; +import org.elasticsearch.index.mapper.internal.RoutingFieldMapper; +import org.elasticsearch.index.mapper.internal.SourceFieldMapper; +import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; +import org.elasticsearch.index.service.IndexService; +import org.elasticsearch.index.shard.IllegalIndexShardStateException; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.service.IndexShard; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.HashMap; +import java.util.Map; + +/** + */ +public class TransportUpdateAction extends TransportInstanceSingleOperationAction { + + private final IndicesService indicesService; + + private final TransportDeleteAction deleteAction; + + private final TransportIndexAction indexAction; + + private final ScriptService scriptService; + + @Inject + public TransportUpdateAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, + IndicesService indicesService, TransportIndexAction indexAction, TransportDeleteAction deleteAction, ScriptService scriptService) { + super(settings, threadPool, clusterService, transportService); + this.indicesService = indicesService; + this.indexAction = indexAction; + this.deleteAction = deleteAction; + this.scriptService = scriptService; + } + + @Override + protected String transportAction() { + return TransportActions.UPDATE; + } + + @Override + protected String executor() { + return ThreadPool.Names.INDEX; + } + + @Override + protected UpdateRequest newRequest() { + return new UpdateRequest(); + } + + @Override + protected UpdateResponse newResponse() { + return new UpdateResponse(); + } + + @Override + protected void checkBlock(UpdateRequest request, ClusterState state) { + state.blocks().indexBlockedRaiseException(ClusterBlockLevel.WRITE, request.index()); + } + + @Override + protected boolean retryOnFailure(Throwable e) { + e = ExceptionsHelper.unwrapCause(e); + if (e instanceof IllegalIndexShardStateException) { + return true; + } + return false; + } + + @Override + protected ShardIterator shards(ClusterState clusterState, UpdateRequest request) throws ElasticSearchException { + if (request.shardId() != -1) { + return clusterState.routingTable().index(request.index()).shard(request.shardId()).primaryShardIt(); + } + ShardIterator shardIterator = clusterService.operationRouting() + .indexShards(clusterService.state(), request.index(), request.type(), request.id(), request.routing()); + ShardRouting shard; + while ((shard = shardIterator.nextOrNull()) != null) { + if (shard.primary()) { + return new PlainShardIterator(shardIterator.shardId(), ImmutableList.of(shard)); + } + } + return new PlainShardIterator(shardIterator.shardId(), ImmutableList.of()); + } + + @Override + protected void shardOperation(final UpdateRequest request, final ActionListener listener) throws ElasticSearchException { + shardOperation(request, listener, 0); + } + + protected void shardOperation(final UpdateRequest request, final ActionListener listener, final int retryCount) throws ElasticSearchException { + IndexService indexService = indicesService.indexServiceSafe(request.index()); + IndexShard indexShard = indexService.shardSafe(request.shardId()); + + GetResult getResult = indexShard.getService().get(request.type(), request.id(), + new String[]{SourceFieldMapper.NAME, RoutingFieldMapper.NAME, ParentFieldMapper.NAME, TimestampFieldMapper.NAME}, true); + + // no doc, what to do, what to do... + if (!getResult.exists()) { + listener.onFailure(new DocumentMissingEngineException(new ShardId(request.index(), request.shardId()), request.type(), request.id())); + return; + } + + if (getResult.internalSourceRef() == null) { + // no source, we can't do nothing, through a failure... + listener.onFailure(new DocumentSourceMissingEngineException(new ShardId(request.index(), request.shardId()), request.type(), request.id())); + return; + } + + Tuple> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef().bytes(), getResult.internalSourceRef().offset(), getResult.internalSourceRef().length(), true); + Map source = sourceAndContent.v2(); + Map ctx = new HashMap(2); + ctx.put("_source", source); + + try { + ExecutableScript script = scriptService.executable(request.scriptLang, request.script, request.scriptParams); + script.setNextVar("ctx", ctx); + script.run(); + // we need to unwrap the ctx... + ctx = (Map) script.unwrap(ctx); + } catch (Exception e) { + throw new ElasticSearchIllegalArgumentException("failed to execute script", e); + } + + String operation = (String) ctx.get("op"); + source = (Map) ctx.get("_source"); + + // apply script to update the source + String routing = getResult.fields().containsKey(RoutingFieldMapper.NAME) ? getResult.field(RoutingFieldMapper.NAME).value().toString() : null; + String parent = getResult.fields().containsKey(ParentFieldMapper.NAME) ? getResult.field(ParentFieldMapper.NAME).value().toString() : null; + // TODO ttl/timestamp + + // TODO percolate? + + // TODO: external version type, does it make sense here? does not seem like it... + + if (operation == null || "index".equals(operation)) { + IndexRequest indexRequest = Requests.indexRequest(request.index()).type(request.type()).id(request.id()).routing(routing).parent(parent) + .source(source, sourceAndContent.v1()) + .version(getResult.version()).replicationType(request.replicationType()).consistencyLevel(request.consistencyLevel()); + indexRequest.operationThreaded(false); + indexAction.execute(indexRequest, new ActionListener() { + @Override + public void onResponse(IndexResponse response) { + UpdateResponse update = new UpdateResponse(response.index(), response.type(), response.id(), response.version()); + listener.onResponse(update); + } + + @Override + public void onFailure(Throwable e) { + e = ExceptionsHelper.unwrapCause(e); + if (e instanceof VersionConflictEngineException) { + if ((retryCount + 1) < request.retryOnConflict()) { + threadPool.executor(executor()).execute(new Runnable() { + @Override + public void run() { + shardOperation(request, listener, retryCount + 1); + } + }); + return; + } + } + } + }); + } else if ("delete".equals(operation)) { + DeleteRequest deleteRequest = Requests.deleteRequest(request.index()).type(request.type()).id(request.id()).routing(routing).parent(parent) + .version(getResult.version()).replicationType(request.replicationType()).consistencyLevel(request.consistencyLevel()); + deleteRequest.operationThreaded(false); + deleteAction.execute(deleteRequest, new ActionListener() { + @Override + public void onResponse(DeleteResponse response) { + UpdateResponse update = new UpdateResponse(response.index(), response.type(), response.id(), response.version()); + listener.onResponse(update); + } + + @Override + public void onFailure(Throwable e) { + e = ExceptionsHelper.unwrapCause(e); + if (e instanceof VersionConflictEngineException) { + if ((retryCount + 1) < request.retryOnConflict()) { + threadPool.executor(executor()).execute(new Runnable() { + @Override + public void run() { + shardOperation(request, listener, retryCount + 1); + } + }); + return; + } + } + listener.onFailure(e); + } + }); + } else if ("none".equals(operation)) { + listener.onResponse(new UpdateResponse(getResult.index(), getResult.type(), getResult.id(), getResult.version())); + } else { + logger.warn("Used update operation [{}] for script [{}], doing nothing...", operation, request.script); + listener.onResponse(new UpdateResponse(getResult.index(), getResult.type(), getResult.id(), getResult.version())); + } + } +} diff --git a/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/src/main/java/org/elasticsearch/action/update/UpdateRequest.java new file mode 100644 index 00000000000..3e1e7a62635 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -0,0 +1,325 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.update; + +import com.google.common.collect.Maps; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.WriteConsistencyLevel; +import org.elasticsearch.action.support.replication.ReplicationType; +import org.elasticsearch.action.support.single.instance.InstanceShardOperationRequest; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.TimeValue; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.action.Actions.addValidationError; + +/** + */ +public class UpdateRequest extends InstanceShardOperationRequest { + + private String type; + private String id; + @Nullable + private String routing; + + String script; + @Nullable + String scriptLang; + @Nullable + Map scriptParams; + + int retryOnConflict = 1; + + private ReplicationType replicationType = ReplicationType.DEFAULT; + private WriteConsistencyLevel consistencyLevel = WriteConsistencyLevel.DEFAULT; + + UpdateRequest() { + + } + + public UpdateRequest(String index, String type, String id) { + this.index = index; + this.type = type; + this.id = id; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = super.validate(); + if (type == null) { + validationException = addValidationError("type is missing", validationException); + } + if (id == null) { + validationException = addValidationError("id is missing", validationException); + } + if (script == null) { + validationException = addValidationError("script is missing", validationException); + } + return validationException; + } + + /** + * Sets the index the document will exists on. + */ + public UpdateRequest index(String index) { + this.index = index; + return this; + } + + /** + * The type of the indexed document. + */ + public String type() { + return type; + } + + /** + * Sets the type of the indexed document. + */ + public UpdateRequest type(String type) { + this.type = type; + return this; + } + + /** + * The id of the indexed document. + */ + public String id() { + return id; + } + + /** + * Sets the id of the indexed document. + */ + public UpdateRequest id(String id) { + this.id = id; + return this; + } + + /** + * Controls the shard routing of the request. Using this value to hash the shard + * and not the id. + */ + public UpdateRequest routing(String routing) { + if (routing != null && routing.length() == 0) { + this.routing = null; + } else { + this.routing = routing; + } + return this; + } + + /** + * Sets the parent id of this document. Will simply set the routing to this value, as it is only + * used for routing with delete requests. + */ + public UpdateRequest parent(String parent) { + if (routing == null) { + routing = parent; + } + return this; + } + + /** + * Controls the shard routing of the request. Using this value to hash the shard + * and not the id. + */ + public String routing() { + return this.routing; + } + + int shardId() { + return this.shardId; + } + + /** + * The script to execute. Note, make sure not to send different script each times and instead + * use script params if possible with the same (automatically compiled) script. + */ + public UpdateRequest script(String script) { + this.script = script; + return this; + } + + /** + * The language of the script to execute. + */ + public UpdateRequest scriptLang(String scriptLang) { + this.scriptLang = scriptLang; + return this; + } + + /** + * Add a script parameter. + */ + public UpdateRequest addScriptParam(String name, Object value) { + if (scriptParams == null) { + scriptParams = Maps.newHashMap(); + } + scriptParams.put(name, value); + return this; + } + + /** + * Sets the script parameters to use with the script. + */ + public UpdateRequest scriptParams(Map scriptParams) { + if (this.scriptParams == null) { + this.scriptParams = scriptParams; + } else { + this.scriptParams.putAll(scriptParams); + } + return this; + } + + /** + * The script to execute. Note, make sure not to send different script each times and instead + * use script params if possible with the same (automatically compiled) script. + */ + public UpdateRequest script(String script, @Nullable Map scriptParams) { + this.script = script; + if (this.scriptParams != null) { + this.scriptParams.putAll(scriptParams); + } else { + this.scriptParams = scriptParams; + } + return this; + } + + /** + * The script to execute. Note, make sure not to send different script each times and instead + * use script params if possible with the same (automatically compiled) script. + * + * @param script The script to execute + * @param scriptLang The script language + * @param scriptParams The script parameters + */ + public UpdateRequest script(String script, @Nullable String scriptLang, @Nullable Map scriptParams) { + this.script = script; + this.scriptLang = scriptLang; + if (this.scriptParams != null) { + this.scriptParams.putAll(scriptParams); + } else { + this.scriptParams = scriptParams; + } + return this; + } + + /** + * Sets the number of retries of a version conflict occurs because the document was updated between + * getting it and updating it. Defaults to 1. + */ + public UpdateRequest retryOnConflict(int retryOnConflict) { + this.retryOnConflict = retryOnConflict; + return this; + } + + public int retryOnConflict() { + return this.retryOnConflict; + } + + /** + * A timeout to wait if the index operation can't be performed immediately. Defaults to 1m. + */ + public UpdateRequest timeout(TimeValue timeout) { + this.timeout = timeout; + return this; + } + + /** + * A timeout to wait if the index operation can't be performed immediately. Defaults to 1m. + */ + public UpdateRequest timeout(String timeout) { + return timeout(TimeValue.parseTimeValue(timeout, null)); + } + + /** + * The replication type. + */ + public ReplicationType replicationType() { + return this.replicationType; + } + + /** + * Sets the replication type. + */ + public UpdateRequest replicationType(ReplicationType replicationType) { + this.replicationType = replicationType; + return this; + } + + public WriteConsistencyLevel consistencyLevel() { + return this.consistencyLevel; + } + + /** + * Sets the consistency level of write. Defaults to {@link org.elasticsearch.action.WriteConsistencyLevel#DEFAULT} + */ + public UpdateRequest consistencyLevel(WriteConsistencyLevel consistencyLevel) { + this.consistencyLevel = consistencyLevel; + return this; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + replicationType = ReplicationType.fromId(in.readByte()); + consistencyLevel = WriteConsistencyLevel.fromId(in.readByte()); + type = in.readUTF(); + id = in.readUTF(); + if (in.readBoolean()) { + routing = in.readUTF(); + } + script = in.readUTF(); + if (in.readBoolean()) { + scriptLang = in.readUTF(); + } + scriptParams = in.readMap(); + retryOnConflict = in.readVInt(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeByte(replicationType.id()); + out.writeByte(consistencyLevel.id()); + out.writeUTF(type); + out.writeUTF(id); + if (routing == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeUTF(routing); + } + out.writeUTF(script); + if (scriptLang == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeUTF(scriptLang); + } + out.writeMap(scriptParams); + out.writeVInt(retryOnConflict); + } +} diff --git a/src/main/java/org/elasticsearch/action/update/UpdateResponse.java b/src/main/java/org/elasticsearch/action/update/UpdateResponse.java new file mode 100644 index 00000000000..f4a41dd9d76 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/update/UpdateResponse.java @@ -0,0 +1,122 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.update; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +/** + */ +public class UpdateResponse implements ActionResponse { + + private String index; + + private String id; + + private String type; + + private long version; + + public UpdateResponse() { + + } + + public UpdateResponse(String index, String type, String id, long version) { + this.index = index; + this.id = id; + this.type = type; + this.version = version; + } + + /** + * The index the document was indexed into. + */ + public String index() { + return this.index; + } + + /** + * The index the document was indexed into. + */ + public String getIndex() { + return index; + } + + /** + * The type of the document indexed. + */ + public String type() { + return this.type; + } + + /** + * The type of the document indexed. + */ + public String getType() { + return type; + } + + /** + * The id of the document indexed. + */ + public String id() { + return this.id; + } + + /** + * The id of the document indexed. + */ + public String getId() { + return id; + } + + /** + * Returns the version of the doc indexed. + */ + public long version() { + return this.version; + } + + /** + * Returns the version of the doc indexed. + */ + public long getVersion() { + return version(); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + index = in.readUTF(); + id = in.readUTF(); + type = in.readUTF(); + version = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeUTF(index); + out.writeUTF(id); + out.writeUTF(type); + out.writeLong(version); + } +} diff --git a/src/main/java/org/elasticsearch/client/Client.java b/src/main/java/org/elasticsearch/client/Client.java index 18050ca8aa1..e72ef413c72 100644 --- a/src/main/java/org/elasticsearch/client/Client.java +++ b/src/main/java/org/elasticsearch/client/Client.java @@ -41,6 +41,8 @@ import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.action.bulk.BulkRequestBuilder; import org.elasticsearch.client.action.count.CountRequestBuilder; import org.elasticsearch.client.action.delete.DeleteRequestBuilder; @@ -52,6 +54,7 @@ import org.elasticsearch.client.action.mlt.MoreLikeThisRequestBuilder; import org.elasticsearch.client.action.percolate.PercolateRequestBuilder; import org.elasticsearch.client.action.search.SearchRequestBuilder; import org.elasticsearch.client.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.client.action.update.UpdateRequestBuilder; import org.elasticsearch.common.Nullable; /** @@ -64,7 +67,6 @@ import org.elasticsearch.common.Nullable; *

A client can either be retrieved from a {@link org.elasticsearch.node.Node} started, or connected remotely * to one or more nodes using {@link org.elasticsearch.client.transport.TransportClient}. * - * * @see org.elasticsearch.node.Node#client() * @see org.elasticsearch.client.transport.TransportClient */ @@ -109,6 +111,32 @@ public interface Client { */ IndexRequestBuilder prepareIndex(); + /** + * Updates a document based on a script. + * + * @param request The update request + * @return The result future + */ + ActionFuture update(UpdateRequest request); + + /** + * Updates a document based on a script. + * + * @param request The update request + * @param listener A listener to be notified with a result + */ + void update(UpdateRequest request, ActionListener listener); + + /** + * Updates a document based on a script. + */ + UpdateRequestBuilder prepareUpdate(); + + /** + * Updates a document based on a script. + */ + UpdateRequestBuilder prepareUpdate(String index, String type, String id); + /** * Index a document associated with a given index and type. *

diff --git a/src/main/java/org/elasticsearch/client/action/update/UpdateRequestBuilder.java b/src/main/java/org/elasticsearch/client/action/update/UpdateRequestBuilder.java new file mode 100644 index 00000000000..dd5c4e3e90d --- /dev/null +++ b/src/main/java/org/elasticsearch/client/action/update/UpdateRequestBuilder.java @@ -0,0 +1,157 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.action.update; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.WriteConsistencyLevel; +import org.elasticsearch.action.support.replication.ReplicationType; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.action.support.BaseRequestBuilder; +import org.elasticsearch.common.unit.TimeValue; + +import java.util.Map; + +/** + */ +public class UpdateRequestBuilder extends BaseRequestBuilder { + + public UpdateRequestBuilder(Client client, String index, String type, String id) { + super(client, new UpdateRequest(index, type, id)); + } + + /** + * Sets the index the document will exists on. + */ + public UpdateRequestBuilder setIndex(String index) { + request.index(index); + return this; + } + + /** + * Sets the type of the indexed document. + */ + public UpdateRequestBuilder setType(String type) { + request.type(type); + return this; + } + + /** + * Sets the id of the indexed document. + */ + public UpdateRequestBuilder setId(String id) { + request.id(id); + return this; + } + + /** + * Controls the shard routing of the request. Using this value to hash the shard + * and not the id. + */ + public UpdateRequestBuilder setRouting(String routing) { + request.routing(routing); + return this; + } + + public UpdateRequestBuilder setParent(String parent) { + request.parent(parent); + return this; + } + + /** + * The script to execute. Note, make sure not to send different script each times and instead + * use script params if possible with the same (automatically compiled) script. + */ + public UpdateRequestBuilder setScript(String script) { + request.script(script); + return this; + } + + /** + * The language of the script to execute. + */ + public UpdateRequestBuilder setScriptLang(String scriptLang) { + request.scriptLang(scriptLang); + return this; + } + + /** + * Sets the script parameters to use with the script. + */ + public UpdateRequestBuilder setScriptParams(Map scriptParams) { + request.scriptParams(scriptParams); + return this; + } + + /** + * Add a script parameter. + */ + public UpdateRequestBuilder addScriptParam(String name, Object value) { + request.addScriptParam(name, value); + return this; + } + + /** + * Sets the number of retries of a version conflict occurs because the document was updated between + * getting it and updating it. Defaults to 1. + */ + public UpdateRequestBuilder setRetryOnConflict(int retryOnConflict) { + request.retryOnConflict(retryOnConflict); + return this; + } + + /** + * A timeout to wait if the index operation can't be performed immediately. Defaults to 1m. + */ + public UpdateRequestBuilder setTimeout(TimeValue timeout) { + request.timeout(timeout); + return this; + } + + /** + * A timeout to wait if the index operation can't be performed immediately. Defaults to 1m. + */ + public UpdateRequestBuilder setTimeout(String timeout) { + request.timeout(timeout); + return this; + } + + /** + * Sets the replication type. + */ + public UpdateRequestBuilder setReplicationType(ReplicationType replicationType) { + request.replicationType(replicationType); + return this; + } + + /** + * Sets the consistency level of write. Defaults to {@link org.elasticsearch.action.WriteConsistencyLevel#DEFAULT} + */ + public UpdateRequestBuilder setConsistencyLevel(WriteConsistencyLevel consistencyLevel) { + request.consistencyLevel(consistencyLevel); + return this; + } + + @Override + protected void doExecute(ActionListener listener) { + client.update(request, listener); + } +} diff --git a/src/main/java/org/elasticsearch/client/node/NodeClient.java b/src/main/java/org/elasticsearch/client/node/NodeClient.java index 74812963a95..8f52e14e8ab 100644 --- a/src/main/java/org/elasticsearch/client/node/NodeClient.java +++ b/src/main/java/org/elasticsearch/client/node/NodeClient.java @@ -43,6 +43,9 @@ import org.elasticsearch.action.percolate.PercolateRequest; import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.percolate.TransportPercolateAction; import org.elasticsearch.action.search.*; +import org.elasticsearch.action.update.TransportUpdateAction; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.internal.InternalClient; import org.elasticsearch.client.support.AbstractClient; @@ -61,6 +64,8 @@ public class NodeClient extends AbstractClient implements InternalClient { private final TransportIndexAction indexAction; + private final TransportUpdateAction updateAction; + private final TransportDeleteAction deleteAction; private final TransportBulkAction bulkAction; @@ -83,13 +88,14 @@ public class NodeClient extends AbstractClient implements InternalClient { @Inject public NodeClient(Settings settings, ThreadPool threadPool, NodeAdminClient admin, - TransportIndexAction indexAction, TransportDeleteAction deleteAction, TransportBulkAction bulkAction, + TransportIndexAction indexAction, TransportUpdateAction updateAction, TransportDeleteAction deleteAction, TransportBulkAction bulkAction, TransportDeleteByQueryAction deleteByQueryAction, TransportGetAction getAction, TransportMultiGetAction multiGetAction, TransportCountAction countAction, TransportSearchAction searchAction, TransportSearchScrollAction searchScrollAction, TransportMoreLikeThisAction moreLikeThisAction, TransportPercolateAction percolateAction) { this.threadPool = threadPool; this.admin = admin; this.indexAction = indexAction; + this.updateAction = updateAction; this.deleteAction = deleteAction; this.bulkAction = bulkAction; this.deleteByQueryAction = deleteByQueryAction; @@ -127,6 +133,16 @@ public class NodeClient extends AbstractClient implements InternalClient { indexAction.execute(request, listener); } + @Override + public ActionFuture update(UpdateRequest request) { + return updateAction.execute(request); + } + + @Override + public void update(UpdateRequest request, ActionListener listener) { + updateAction.execute(request, listener); + } + @Override public ActionFuture delete(DeleteRequest request) { return deleteAction.execute(request); diff --git a/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/src/main/java/org/elasticsearch/client/support/AbstractClient.java index 38340d8f36d..4012ad515db 100644 --- a/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -30,6 +30,7 @@ import org.elasticsearch.client.action.mlt.MoreLikeThisRequestBuilder; import org.elasticsearch.client.action.percolate.PercolateRequestBuilder; import org.elasticsearch.client.action.search.SearchRequestBuilder; import org.elasticsearch.client.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.client.action.update.UpdateRequestBuilder; import org.elasticsearch.client.internal.InternalClient; import org.elasticsearch.common.Nullable; @@ -53,6 +54,16 @@ public abstract class AbstractClient implements InternalClient { return prepareIndex().setIndex(index).setType(type).setId(id); } + @Override + public UpdateRequestBuilder prepareUpdate() { + return new UpdateRequestBuilder(this, null, null, null); + } + + @Override + public UpdateRequestBuilder prepareUpdate(String index, String type, String id) { + return new UpdateRequestBuilder(this, index, type, id); + } + @Override public DeleteRequestBuilder prepareDelete() { return new DeleteRequestBuilder(this, null); diff --git a/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/src/main/java/org/elasticsearch/client/transport/TransportClient.java index 890ad1f3b38..c13d0d9ff6c 100644 --- a/src/main/java/org/elasticsearch/client/transport/TransportClient.java +++ b/src/main/java/org/elasticsearch/client/transport/TransportClient.java @@ -43,6 +43,8 @@ import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.client.transport.action.ClientTransportActionModule; @@ -80,8 +82,6 @@ import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilde *

*

The transport client important modules used is the {@link org.elasticsearch.transport.TransportModule} which is * started in client mode (only connects, no bind). - * - * */ public class TransportClient extends AbstractClient { @@ -259,6 +259,16 @@ public class TransportClient extends AbstractClient { internalClient.index(request, listener); } + @Override + public ActionFuture update(UpdateRequest request) { + return internalClient.update(request); + } + + @Override + public void update(UpdateRequest request, ActionListener listener) { + internalClient.update(request, listener); + } + @Override public ActionFuture delete(DeleteRequest request) { return internalClient.delete(request); diff --git a/src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java b/src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java index a8b7fdac8ee..1db0d2f5053 100644 --- a/src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java +++ b/src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java @@ -61,6 +61,7 @@ import org.elasticsearch.client.transport.action.index.ClientTransportIndexActio import org.elasticsearch.client.transport.action.percolate.ClientTransportPercolateAction; import org.elasticsearch.client.transport.action.search.ClientTransportSearchAction; import org.elasticsearch.client.transport.action.search.ClientTransportSearchScrollAction; +import org.elasticsearch.client.transport.action.update.ClientTransportUpdateAction; import org.elasticsearch.common.inject.AbstractModule; /** @@ -80,6 +81,7 @@ public class ClientTransportActionModule extends AbstractModule { bind(ClientTransportSearchScrollAction.class).asEagerSingleton(); bind(ClientTransportBulkAction.class).asEagerSingleton(); bind(ClientTransportPercolateAction.class).asEagerSingleton(); + bind(ClientTransportUpdateAction.class).asEagerSingleton(); bind(ClientTransportIndicesExistsAction.class).asEagerSingleton(); bind(ClientTransportIndicesStatsAction.class).asEagerSingleton(); diff --git a/src/main/java/org/elasticsearch/client/transport/action/update/ClientTransportUpdateAction.java b/src/main/java/org/elasticsearch/client/transport/action/update/ClientTransportUpdateAction.java new file mode 100644 index 00000000000..deec84c62a3 --- /dev/null +++ b/src/main/java/org/elasticsearch/client/transport/action/update/ClientTransportUpdateAction.java @@ -0,0 +1,44 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.transport.action.update; + +import org.elasticsearch.action.TransportActions; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.transport.TransportService; + +/** + * + */ +public class ClientTransportUpdateAction extends BaseClientTransportAction { + + @Inject + public ClientTransportUpdateAction(Settings settings, TransportService transportService) { + super(settings, transportService, UpdateResponse.class); + } + + @Override + protected String action() { + return TransportActions.UPDATE; + } +} diff --git a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java index 6f9370c4c61..526e1bbe3e5 100644 --- a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java +++ b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java @@ -42,6 +42,8 @@ import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.internal.InternalClient; import org.elasticsearch.client.support.AbstractClient; @@ -57,6 +59,7 @@ import org.elasticsearch.client.transport.action.mlt.ClientTransportMoreLikeThis import org.elasticsearch.client.transport.action.percolate.ClientTransportPercolateAction; import org.elasticsearch.client.transport.action.search.ClientTransportSearchAction; import org.elasticsearch.client.transport.action.search.ClientTransportSearchScrollAction; +import org.elasticsearch.client.transport.action.update.ClientTransportUpdateAction; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -75,6 +78,8 @@ public class InternalTransportClient extends AbstractClient implements InternalC private final ClientTransportIndexAction indexAction; + private final ClientTransportUpdateAction updateAction; + private final ClientTransportDeleteAction deleteAction; private final ClientTransportBulkAction bulkAction; @@ -98,7 +103,7 @@ public class InternalTransportClient extends AbstractClient implements InternalC @Inject public InternalTransportClient(Settings settings, ThreadPool threadPool, TransportClientNodesService nodesService, InternalTransportAdminClient adminClient, - ClientTransportIndexAction indexAction, ClientTransportDeleteAction deleteAction, ClientTransportBulkAction bulkAction, ClientTransportGetAction getAction, ClientTransportMultiGetAction multiGetAction, + ClientTransportIndexAction indexAction, ClientTransportUpdateAction updateAction, ClientTransportDeleteAction deleteAction, ClientTransportBulkAction bulkAction, ClientTransportGetAction getAction, ClientTransportMultiGetAction multiGetAction, ClientTransportDeleteByQueryAction deleteByQueryAction, ClientTransportCountAction countAction, ClientTransportSearchAction searchAction, ClientTransportSearchScrollAction searchScrollAction, ClientTransportMoreLikeThisAction moreLikeThisAction, ClientTransportPercolateAction percolateAction) { @@ -107,6 +112,7 @@ public class InternalTransportClient extends AbstractClient implements InternalC this.adminClient = adminClient; this.indexAction = indexAction; + this.updateAction = updateAction; this.deleteAction = deleteAction; this.bulkAction = bulkAction; this.getAction = getAction; @@ -154,6 +160,26 @@ public class InternalTransportClient extends AbstractClient implements InternalC }, listener); } + @Override + public ActionFuture update(final UpdateRequest request) { + return nodesService.execute(new TransportClientNodesService.NodeCallback>() { + @Override + public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { + return updateAction.execute(node, request); + } + }); + } + + @Override + public void update(final UpdateRequest request, final ActionListener listener) { + nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { + @Override + public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { + updateAction.execute(node, request, listener); + } + }, listener); + } + @Override public ActionFuture delete(final DeleteRequest request) { return nodesService.execute(new TransportClientNodesService.NodeCallback>() { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java index d0babc944b7..7d6e38bf782 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java @@ -342,7 +342,7 @@ public class MappingMetaData { * Converts the serialized compressed form of the mappings into a parsed map. */ public Map sourceAsMap() throws IOException { - Map mapping = XContentHelper.convertToMap(source.compressed(), 0, source.compressed().length).v2(); + Map mapping = XContentHelper.convertToMap(source.compressed(), 0, source.compressed().length, true).v2(); if (mapping.size() == 1 && mapping.containsKey(type())) { // the type name is the root value, reduce it mapping = (Map) mapping.get(type()); diff --git a/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index 1aacc1c787a..66f2a114b9d 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -19,9 +19,12 @@ package org.elasticsearch.common.io.stream; +import org.elasticsearch.common.Nullable; + import java.io.IOException; import java.io.InputStream; import java.io.UTFDataFormatException; +import java.util.*; /** * @@ -249,4 +252,65 @@ public abstract class StreamInput extends InputStream { // readBytes(b, off, len); // return len; // } + + public + @Nullable + Map readMap() throws IOException { + return (Map) readFieldValue(); + } + + @SuppressWarnings({"unchecked"}) + private + @Nullable + Object readFieldValue() throws IOException { + byte type = readByte(); + if (type == -1) { + return null; + } else if (type == 0) { + return readUTF(); + } else if (type == 1) { + return readInt(); + } else if (type == 2) { + return readLong(); + } else if (type == 3) { + return readFloat(); + } else if (type == 4) { + return readDouble(); + } else if (type == 5) { + return readBoolean(); + } else if (type == 6) { + int bytesSize = readVInt(); + byte[] value = new byte[bytesSize]; + readFully(value); + return value; + } else if (type == 7) { + int size = readVInt(); + List list = new ArrayList(size); + for (int i = 0; i < size; i++) { + list.add(readFieldValue()); + } + return list; + } else if (type == 8) { + int size = readVInt(); + Object[] list = new Object[size]; + for (int i = 0; i < size; i++) { + list[i] = readFieldValue(); + } + return list; + } else if (type == 9 || type == 10) { + int size = readVInt(); + Map map; + if (type == 9) { + map = new LinkedHashMap(size); + } else { + map = new HashMap(size); + } + for (int i = 0; i < size; i++) { + map.put(readUTF(), readFieldValue()); + } + return map; + } else { + throw new IOException("Can't read unknown type [" + type + "]"); + } + } } diff --git a/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index 31705e08478..704ab40f469 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -19,8 +19,13 @@ package org.elasticsearch.common.io.stream; +import org.elasticsearch.common.Nullable; + import java.io.IOException; import java.io.OutputStream; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; /** * @@ -209,4 +214,67 @@ public abstract class StreamOutput extends OutputStream { public void write(byte[] b, int off, int len) throws IOException { writeBytes(b, off, len); } + + public void writeMap(@Nullable Map map) throws IOException { + writeValue(map); + } + + private void writeValue(@Nullable Object value) throws IOException { + if (value == null) { + writeByte((byte) -1); + return; + } + Class type = value.getClass(); + if (type == String.class) { + writeByte((byte) 0); + writeUTF((String) value); + } else if (type == Integer.class) { + writeByte((byte) 1); + writeInt((Integer) value); + } else if (type == Long.class) { + writeByte((byte) 2); + writeLong((Long) value); + } else if (type == Float.class) { + writeByte((byte) 3); + writeFloat((Float) value); + } else if (type == Double.class) { + writeByte((byte) 4); + writeDouble((Double) value); + } else if (type == Boolean.class) { + writeByte((byte) 5); + writeBoolean((Boolean) value); + } else if (type == byte[].class) { + writeByte((byte) 6); + writeVInt(((byte[]) value).length); + writeBytes(((byte[]) value)); + } else if (value instanceof List) { + writeByte((byte) 7); + List list = (List) value; + writeVInt(list.size()); + for (Object o : list) { + writeValue(o); + } + } else if (value instanceof Object[]) { + writeByte((byte) 8); + Object[] list = (Object[]) value; + writeVInt(list.length); + for (Object o : list) { + writeValue(o); + } + } else if (value instanceof Map) { + if (value instanceof LinkedHashMap) { + writeByte((byte) 9); + } else { + writeByte((byte) 10); + } + Map map = (Map) value; + writeVInt(map.size()); + for (Map.Entry entry : map.entrySet()) { + writeUTF(entry.getKey()); + writeValue(entry.getValue()); + } + } else { + throw new IOException("Can't write type [" + type + "]"); + } + } } diff --git a/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index d48520791a7..75b70983afd 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -50,17 +50,24 @@ public class XContentHelper { } } - public static Tuple> convertToMap(byte[] data, int offset, int length) throws ElasticSearchParseException { + public static Tuple> convertToMap(byte[] data, int offset, int length, boolean ordered) throws ElasticSearchParseException { try { + XContentParser parser; + XContentType contentType; if (LZF.isCompressed(data, offset, length)) { BytesStreamInput siBytes = new BytesStreamInput(data, offset, length); LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes); - XContentType contentType = XContentFactory.xContentType(siLzf); + contentType = XContentFactory.xContentType(siLzf); siLzf.resetToBufferStart(); - return Tuple.create(contentType, XContentFactory.xContent(contentType).createParser(siLzf).mapAndClose()); + parser = XContentFactory.xContent(contentType).createParser(siLzf); } else { - XContentType contentType = XContentFactory.xContentType(data, offset, length); - return Tuple.create(contentType, XContentFactory.xContent(contentType).createParser(data, offset, length).mapAndClose()); + contentType = XContentFactory.xContentType(data, offset, length); + parser = XContentFactory.xContent(contentType).createParser(data, offset, length); + } + if (ordered) { + return Tuple.create(contentType, parser.mapOrderedAndClose()); + } else { + return Tuple.create(contentType, parser.mapAndClose()); } } catch (IOException e) { throw new ElasticSearchParseException("Failed to parse content to map", e); diff --git a/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java b/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java index 227855f4ca0..95e326abf89 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java +++ b/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java @@ -122,6 +122,8 @@ public interface XContentParser { Map mapAndClose() throws IOException; + Map mapOrderedAndClose() throws IOException; + String text() throws IOException; String textOrNull() throws IOException; diff --git a/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java b/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java index 300fbf20f78..080df1c060f 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java +++ b/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java @@ -124,4 +124,13 @@ public abstract class AbstractXContentParser implements XContentParser { close(); } } + + @Override + public Map mapOrderedAndClose() throws IOException { + try { + return mapOrdered(); + } finally { + close(); + } + } } diff --git a/src/main/java/org/elasticsearch/index/engine/DocumentMIssingEngineException.java b/src/main/java/org/elasticsearch/index/engine/DocumentMIssingEngineException.java new file mode 100644 index 00000000000..1bd3dda7b70 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/engine/DocumentMIssingEngineException.java @@ -0,0 +1,38 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.engine; + +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.rest.RestStatus; + +/** + * + */ +public class DocumentMissingEngineException extends EngineException { + + public DocumentMissingEngineException(ShardId shardId, String type, String id) { + super(shardId, "[" + type + "][" + id + "]: document missing"); + } + + @Override + public RestStatus status() { + return RestStatus.CONFLICT; + } +} diff --git a/src/main/java/org/elasticsearch/index/engine/DocumentSourceMissingEngineException.java b/src/main/java/org/elasticsearch/index/engine/DocumentSourceMissingEngineException.java new file mode 100644 index 00000000000..a918632eefd --- /dev/null +++ b/src/main/java/org/elasticsearch/index/engine/DocumentSourceMissingEngineException.java @@ -0,0 +1,38 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.engine; + +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.rest.RestStatus; + +/** + * + */ +public class DocumentSourceMissingEngineException extends EngineException { + + public DocumentSourceMissingEngineException(ShardId shardId, String type, String id) { + super(shardId, "[" + type + "][" + id + "]: document source missing"); + } + + @Override + public RestStatus status() { + return RestStatus.BAD_REQUEST; + } +} diff --git a/src/main/java/org/elasticsearch/index/get/GetResult.java b/src/main/java/org/elasticsearch/index/get/GetResult.java index 028381d0f60..ef15d4089be 100644 --- a/src/main/java/org/elasticsearch/index/get/GetResult.java +++ b/src/main/java/org/elasticsearch/index/get/GetResult.java @@ -179,6 +179,13 @@ public class GetResult implements Streamable, Iterable, ToXContent { return this.source; } + /** + * Internal source representation, might be compressed.... + */ + public BytesHolder internalSourceRef() { + return source; + } + /** * Is the source empty (not available) or not. */ diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java index b5596fc6bdd..34ac7473b86 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java @@ -230,7 +230,7 @@ public class SourceFieldMapper extends AbstractFieldMapper implements In if (filtered) { // we don't update the context source if we filter, we want to keep it as is... - Tuple> mapTuple = XContentHelper.convertToMap(data, dataOffset, dataLength); + Tuple> mapTuple = XContentHelper.convertToMap(data, dataOffset, dataLength, true); Map filteredSource = XContentMapValues.filter(mapTuple.v2(), includes, excludes); CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry(); StreamOutput streamOutput; diff --git a/src/main/java/org/elasticsearch/rest/action/RestActionModule.java b/src/main/java/org/elasticsearch/rest/action/RestActionModule.java index b007b1738a5..c90fadf63cc 100644 --- a/src/main/java/org/elasticsearch/rest/action/RestActionModule.java +++ b/src/main/java/org/elasticsearch/rest/action/RestActionModule.java @@ -71,6 +71,7 @@ import org.elasticsearch.rest.action.mlt.RestMoreLikeThisAction; import org.elasticsearch.rest.action.percolate.RestPercolateAction; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.rest.action.search.RestSearchScrollAction; +import org.elasticsearch.rest.action.update.RestUpdateAction; import java.util.List; @@ -137,24 +138,20 @@ public class RestActionModule extends AbstractModule { bind(RestClearIndicesCacheAction.class).asEagerSingleton(); bind(RestIndexAction.class).asEagerSingleton(); - bind(RestGetAction.class).asEagerSingleton(); bind(RestMultiGetAction.class).asEagerSingleton(); - bind(RestDeleteAction.class).asEagerSingleton(); - bind(RestDeleteByQueryAction.class).asEagerSingleton(); - bind(RestCountAction.class).asEagerSingleton(); bind(RestBulkAction.class).asEagerSingleton(); + bind(RestUpdateAction.class).asEagerSingleton(); + bind(RestPercolateAction.class).asEagerSingleton(); bind(RestSearchAction.class).asEagerSingleton(); bind(RestSearchScrollAction.class).asEagerSingleton(); - + bind(RestValidateQueryAction.class).asEagerSingleton(); bind(RestMoreLikeThisAction.class).asEagerSingleton(); - - bind(RestPercolateAction.class).asEagerSingleton(); } } diff --git a/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java b/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java new file mode 100644 index 00000000000..e6bef6050b1 --- /dev/null +++ b/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java @@ -0,0 +1,146 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.update; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.WriteConsistencyLevel; +import org.elasticsearch.action.support.replication.ReplicationType; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.*; +import org.elasticsearch.rest.action.support.RestXContentBuilder; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.rest.RestStatus.CREATED; +import static org.elasticsearch.rest.RestStatus.OK; + +/** + */ +public class RestUpdateAction extends BaseRestHandler { + + @Inject + public RestUpdateAction(Settings settings, Client client, RestController controller) { + super(settings, client); + controller.registerHandler(POST, "/{index}/{type}/{id}/_update", this); + } + + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + UpdateRequest updateRequest = new UpdateRequest(request.param("index"), request.param("type"), request.param("id")); + updateRequest.routing(request.param("routing")); + updateRequest.parent(request.param("parent")); // order is important, set it after routing, so it will set the routing + updateRequest.timeout(request.paramAsTime("timeout", updateRequest.timeout())); + String replicationType = request.param("replication"); + if (replicationType != null) { + updateRequest.replicationType(ReplicationType.fromString(replicationType)); + } + String consistencyLevel = request.param("consistency"); + if (consistencyLevel != null) { + updateRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel)); + } + // we just send a response, no need to fork + updateRequest.listenerThreaded(false); + updateRequest.script(request.param("script")); + updateRequest.scriptLang(request.param("lang")); + for (Map.Entry entry : request.params().entrySet()) { + if (entry.getKey().startsWith("sp_")) { + updateRequest.addScriptParam(entry.getKey().substring(3), entry.getValue()); + } + } + updateRequest.retryOnConflict(request.paramAsInt("retry_on_conflict", updateRequest.retryOnConflict())); + + // see if we have it in the body + if (request.hasContent()) { + XContentType xContentType = XContentFactory.xContentType(request.contentByteArray(), request.contentByteArrayOffset(), request.contentLength()); + if (xContentType != null) { + try { + Map content = XContentFactory.xContent(xContentType) + .createParser(request.contentByteArray(), request.contentByteArrayOffset(), request.contentLength()).mapAndClose(); + if (content.containsKey("script")) { + updateRequest.script(content.get("script").toString()); + } + if (content.containsKey("lang")) { + updateRequest.scriptLang(content.get("lang").toString()); + } + if (content.containsKey("params")) { + updateRequest.scriptParams((Map) content.get("params")); + } + } catch (Exception e) { + try { + channel.sendResponse(new XContentThrowableRestResponse(request, e)); + } catch (IOException e1) { + logger.warn("Failed to send response", e1); + } + return; + } + } + } + + client.update(updateRequest, new ActionListener() { + @Override + public void onResponse(UpdateResponse response) { + try { + XContentBuilder builder = RestXContentBuilder.restContentBuilder(request); + builder.startObject() + .field(Fields.OK, true) + .field(Fields._INDEX, response.index()) + .field(Fields._TYPE, response.type()) + .field(Fields._ID, response.id()) + .field(Fields._VERSION, response.version()); + builder.endObject(); + RestStatus status = OK; + if (response.version() == 1) { + status = CREATED; + } + channel.sendResponse(new XContentRestResponse(request, status, builder)); + } catch (Exception e) { + onFailure(e); + } + } + + @Override + public void onFailure(Throwable e) { + try { + channel.sendResponse(new XContentThrowableRestResponse(request, e)); + } catch (IOException e1) { + logger.error("Failed to send failure response", e1); + } + } + }); + } + + static final class Fields { + static final XContentBuilderString OK = new XContentBuilderString("ok"); + static final XContentBuilderString _INDEX = new XContentBuilderString("_index"); + static final XContentBuilderString _TYPE = new XContentBuilderString("_type"); + static final XContentBuilderString _ID = new XContentBuilderString("_id"); + static final XContentBuilderString _VERSION = new XContentBuilderString("_version"); + } +} diff --git a/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index 125cc6a6149..54e46378877 100644 --- a/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -76,7 +76,7 @@ public class SourceLookup implements Map { } public static Map sourceAsMap(byte[] bytes, int offset, int length) throws ElasticSearchParseException { - return XContentHelper.convertToMap(bytes, offset, length).v2(); + return XContentHelper.convertToMap(bytes, offset, length, false).v2(); } public void setNextReader(IndexReader reader) { diff --git a/src/test/java/org/elasticsearch/test/integration/client/transport/DiscoveryTransportClientTests.yml b/src/test/java/org/elasticsearch/test/integration/client/transport/DiscoveryTransportClientTests.yml deleted file mode 100644 index 641d322a2b3..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/client/transport/DiscoveryTransportClientTests.yml +++ /dev/null @@ -1,6 +0,0 @@ -cluster: - routing: - schedule: 100ms -index: - number_of_shards: 5 - number_of_replicas: 1 diff --git a/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientDocumentActionsTests.java b/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientDocumentActionsTests.java index bed87ea860e..e4a4841a006 100644 --- a/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientDocumentActionsTests.java +++ b/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientDocumentActionsTests.java @@ -22,6 +22,8 @@ package org.elasticsearch.test.integration.client.transport; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.network.NetworkUtils; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.node.internal.InternalNode; import org.elasticsearch.test.integration.document.DocumentActionsTests; @@ -38,6 +40,7 @@ public class TransportClientDocumentActionsTests extends DocumentActionsTests { protected Client getClient1() { TransportAddress server1Address = ((InternalNode) node("server1")).injector().getInstance(TransportService.class).boundAddress().publishAddress(); TransportClient client = new TransportClient(settingsBuilder() + .put(nodeSettings()) .put("cluster.name", "test-cluster-" + NetworkUtils.getLocalAddress().getHostName()) .put("client.transport.sniff", false).build()); client.addTransportAddress(server1Address); @@ -48,9 +51,15 @@ public class TransportClientDocumentActionsTests extends DocumentActionsTests { protected Client getClient2() { TransportAddress server2Address = ((InternalNode) node("server2")).injector().getInstance(TransportService.class).boundAddress().publishAddress(); TransportClient client = new TransportClient(settingsBuilder() + .put(nodeSettings()) .put("cluster.name", "test-cluster-" + NetworkUtils.getLocalAddress().getHostName()) .put("client.transport.sniff", false).build()); client.addTransportAddress(server2Address); return client; } + + @Override + protected Settings nodeSettings() { + return ImmutableSettings.settingsBuilder().put("client.transport.nodes_sampler_interval", "30s").build(); + } } diff --git a/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientDocumentActionsTests.yml b/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientDocumentActionsTests.yml deleted file mode 100644 index f137ce572b9..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientDocumentActionsTests.yml +++ /dev/null @@ -1,12 +0,0 @@ -cluster: - routing: - schedule: 100ms -index: - number_of_shards: 5 - number_of_replicas: 1 - -# use large interval node sampler -client: - transport: - nodes_sampler_interval: 30s - diff --git a/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientMoreLikeThisActionTests.java b/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientMoreLikeThisActionTests.java deleted file mode 100644 index a81140983ab..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientMoreLikeThisActionTests.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.integration.client.transport; - -import org.elasticsearch.client.Client; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.network.NetworkUtils; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.node.internal.InternalNode; -import org.elasticsearch.test.integration.document.MoreLikeThisActionTests; -import org.elasticsearch.transport.TransportService; - -import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; - -/** - * - */ -public class TransportClientMoreLikeThisActionTests extends MoreLikeThisActionTests { - - @Override - protected Client getClient1() { - TransportAddress server1Address = ((InternalNode) node("server1")).injector().getInstance(TransportService.class).boundAddress().publishAddress(); - TransportClient client = new TransportClient(settingsBuilder() - .put("cluster.name", "test-cluster-" + NetworkUtils.getLocalAddress().getHostName()) - .put("discovery.enabled", false).build()); - client.addTransportAddress(server1Address); - return client; - } - - @Override - protected Client getClient2() { - TransportAddress server1Address = ((InternalNode) node("server2")).injector().getInstance(TransportService.class).boundAddress().publishAddress(); - TransportClient client = new TransportClient(settingsBuilder() - .put("cluster.name", "test-cluster-" + NetworkUtils.getLocalAddress().getHostName()) - .put("discovery.enabled", false).build()); - client.addTransportAddress(server1Address); - return client; - } -} diff --git a/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientMoreLikeThisActionTests.yml b/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientMoreLikeThisActionTests.yml deleted file mode 100644 index 641d322a2b3..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientMoreLikeThisActionTests.yml +++ /dev/null @@ -1,6 +0,0 @@ -cluster: - routing: - schedule: 100ms -index: - number_of_shards: 5 - number_of_replicas: 1 diff --git a/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientSniffDocumentActionsTests.java b/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientSniffDocumentActionsTests.java index 0afba385fdb..326ddf040bd 100644 --- a/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientSniffDocumentActionsTests.java +++ b/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientSniffDocumentActionsTests.java @@ -22,6 +22,8 @@ package org.elasticsearch.test.integration.client.transport; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.network.NetworkUtils; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.node.internal.InternalNode; import org.elasticsearch.test.integration.document.DocumentActionsTests; @@ -38,6 +40,7 @@ public class TransportClientSniffDocumentActionsTests extends DocumentActionsTes protected Client getClient1() { TransportAddress server1Address = ((InternalNode) node("server1")).injector().getInstance(TransportService.class).boundAddress().publishAddress(); TransportClient client = new TransportClient(settingsBuilder() + .put(nodeSettings()) .put("cluster.name", "test-cluster-" + NetworkUtils.getLocalAddress().getHostName()) .put("client.transport.sniff", true).build()); client.addTransportAddress(server1Address); @@ -48,9 +51,15 @@ public class TransportClientSniffDocumentActionsTests extends DocumentActionsTes protected Client getClient2() { TransportAddress server2Address = ((InternalNode) node("server2")).injector().getInstance(TransportService.class).boundAddress().publishAddress(); TransportClient client = new TransportClient(settingsBuilder() + .put(nodeSettings()) .put("cluster.name", "test-cluster-" + NetworkUtils.getLocalAddress().getHostName()) .put("client.transport.sniff", true).build()); client.addTransportAddress(server2Address); return client; } + + @Override + protected Settings nodeSettings() { + return ImmutableSettings.settingsBuilder().put("client.transport.nodes_sampler_interval", "30s").build(); + } } diff --git a/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientSniffDocumentActionsTests.yml b/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientSniffDocumentActionsTests.yml deleted file mode 100644 index f137ce572b9..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientSniffDocumentActionsTests.yml +++ /dev/null @@ -1,12 +0,0 @@ -cluster: - routing: - schedule: 100ms -index: - number_of_shards: 5 - number_of_replicas: 1 - -# use large interval node sampler -client: - transport: - nodes_sampler_interval: 30s - diff --git a/src/test/java/org/elasticsearch/test/integration/document/AliasedIndexDocumentActionsTests.java b/src/test/java/org/elasticsearch/test/integration/document/AliasedIndexDocumentActionsTests.java index 1091f4bc3a5..8a01e365a1f 100644 --- a/src/test/java/org/elasticsearch/test/integration/document/AliasedIndexDocumentActionsTests.java +++ b/src/test/java/org/elasticsearch/test/integration/document/AliasedIndexDocumentActionsTests.java @@ -19,6 +19,9 @@ package org.elasticsearch.test.integration.document; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; + import static org.elasticsearch.client.Requests.createIndexRequest; import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; @@ -42,4 +45,9 @@ public class AliasedIndexDocumentActionsTests extends DocumentActionsTests { protected String getConcreteIndexName() { return "test1"; } + + @Override + protected Settings nodeSettings() { + return ImmutableSettings.settingsBuilder().put("action.auto_create_index", false).build(); + } } \ No newline at end of file diff --git a/src/test/java/org/elasticsearch/test/integration/document/AliasedIndexDocumentActionsTests.yml b/src/test/java/org/elasticsearch/test/integration/document/AliasedIndexDocumentActionsTests.yml deleted file mode 100644 index 91495fb3cf6..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/document/AliasedIndexDocumentActionsTests.yml +++ /dev/null @@ -1,8 +0,0 @@ -cluster: - routing: - schedule: 100ms -index: - number_of_shards: 5 - number_of_replicas: 1 -action: - auto_create_index: false \ No newline at end of file diff --git a/src/test/java/org/elasticsearch/test/integration/document/DocumentActionsTests.java b/src/test/java/org/elasticsearch/test/integration/document/DocumentActionsTests.java index a81dc19f5a0..d1411bc4e1b 100644 --- a/src/test/java/org/elasticsearch/test/integration/document/DocumentActionsTests.java +++ b/src/test/java/org/elasticsearch/test/integration/document/DocumentActionsTests.java @@ -34,10 +34,14 @@ import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; import org.elasticsearch.action.support.replication.ReplicationType; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.Unicode; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.engine.DocumentMissingEngineException; import org.elasticsearch.test.integration.AbstractNodesTests; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -62,8 +66,8 @@ public class DocumentActionsTests extends AbstractNodesTests { @BeforeClass public void startNodes() { - startNode("server1"); - startNode("server2"); + startNode("server1", nodeSettings()); + startNode("server2", nodeSettings()); client1 = getClient1(); client2 = getClient2(); @@ -87,6 +91,10 @@ public class DocumentActionsTests extends AbstractNodesTests { client1.admin().indices().create(createIndexRequest("test")).actionGet(); } + protected Settings nodeSettings() { + return ImmutableSettings.Builder.EMPTY_SETTINGS; + } + protected String getConcreteIndexName() { return "test"; } @@ -261,6 +269,57 @@ public class DocumentActionsTests extends AbstractNodesTests { } } + @Test + public void testUpdate() throws Exception { + createIndex(); + ClusterHealthResponse clusterHealth = client1.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); + assertThat(clusterHealth.timedOut(), equalTo(false)); + assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.GREEN)); + + try { + client1.prepareUpdate("test", "type1", "1").setScript("ctx._source.field++").execute().actionGet(); + assert false; + } catch (DocumentMissingEngineException e) { + // all is well + } + + client1.prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet(); + + UpdateResponse updateResponse = client1.prepareUpdate("test", "type1", "1").setScript("ctx._source.field += 1").execute().actionGet(); + assertThat(updateResponse.version(), equalTo(2L)); + + for (int i = 0; i < 5; i++) { + GetResponse getResponse = client1.prepareGet("test", "type1", "1").execute().actionGet(); + assertThat(getResponse.sourceAsMap().get("field").toString(), equalTo("2")); + } + + updateResponse = client1.prepareUpdate("test", "type1", "1").setScript("ctx._source.field += count").addScriptParam("count", 3).execute().actionGet(); + assertThat(updateResponse.version(), equalTo(3L)); + + for (int i = 0; i < 5; i++) { + GetResponse getResponse = client1.prepareGet("test", "type1", "1").execute().actionGet(); + assertThat(getResponse.sourceAsMap().get("field").toString(), equalTo("5")); + } + + // check noop + updateResponse = client1.prepareUpdate("test", "type1", "1").setScript("ctx.op = 'none'").execute().actionGet(); + assertThat(updateResponse.version(), equalTo(3L)); + + for (int i = 0; i < 5; i++) { + GetResponse getResponse = client1.prepareGet("test", "type1", "1").execute().actionGet(); + assertThat(getResponse.sourceAsMap().get("field").toString(), equalTo("5")); + } + + // check delete + updateResponse = client1.prepareUpdate("test", "type1", "1").setScript("ctx.op = 'delete'").execute().actionGet(); + assertThat(updateResponse.version(), equalTo(4L)); + + for (int i = 0; i < 5; i++) { + GetResponse getResponse = client1.prepareGet("test", "type1", "1").execute().actionGet(); + assertThat(getResponse.exists(), equalTo(false)); + } + } + @Test public void testBulk() throws Exception { createIndex(); diff --git a/src/test/java/org/elasticsearch/test/integration/document/DocumentActionsTests.yml b/src/test/java/org/elasticsearch/test/integration/document/DocumentActionsTests.yml deleted file mode 100644 index 641d322a2b3..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/document/DocumentActionsTests.yml +++ /dev/null @@ -1,6 +0,0 @@ -cluster: - routing: - schedule: 100ms -index: - number_of_shards: 5 - number_of_replicas: 1 diff --git a/src/test/java/org/elasticsearch/test/integration/document/LocalDocumentActionsTests.java b/src/test/java/org/elasticsearch/test/integration/document/LocalDocumentActionsTests.java index ac14ad31c70..a16abe24e6d 100644 --- a/src/test/java/org/elasticsearch/test/integration/document/LocalDocumentActionsTests.java +++ b/src/test/java/org/elasticsearch/test/integration/document/LocalDocumentActionsTests.java @@ -19,8 +19,16 @@ package org.elasticsearch.test.integration.document; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; + /** * */ public class LocalDocumentActionsTests extends DocumentActionsTests { + + @Override + protected Settings nodeSettings() { + return ImmutableSettings.settingsBuilder().put("node.local", true).build(); + } } diff --git a/src/test/java/org/elasticsearch/test/integration/document/LocalDocumentActionsTests.yml b/src/test/java/org/elasticsearch/test/integration/document/LocalDocumentActionsTests.yml deleted file mode 100644 index 8d487363368..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/document/LocalDocumentActionsTests.yml +++ /dev/null @@ -1,8 +0,0 @@ -node: - local: true -cluster: - routing: - schedule: 100ms -index: - number_of_shards: 5 - number_of_replicas: 1 diff --git a/src/test/java/org/elasticsearch/test/integration/document/MoreLikeThisActionTests.yml b/src/test/java/org/elasticsearch/test/integration/document/MoreLikeThisActionTests.yml deleted file mode 100644 index 641d322a2b3..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/document/MoreLikeThisActionTests.yml +++ /dev/null @@ -1,6 +0,0 @@ -cluster: - routing: - schedule: 100ms -index: - number_of_shards: 5 - number_of_replicas: 1 diff --git a/src/test/java/org/elasticsearch/test/integration/document/GetActionTests.java b/src/test/java/org/elasticsearch/test/integration/get/GetActionTests.java similarity index 99% rename from src/test/java/org/elasticsearch/test/integration/document/GetActionTests.java rename to src/test/java/org/elasticsearch/test/integration/get/GetActionTests.java index 874066135a2..15c3237698b 100644 --- a/src/test/java/org/elasticsearch/test/integration/document/GetActionTests.java +++ b/src/test/java/org/elasticsearch/test/integration/get/GetActionTests.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.test.integration.document; +package org.elasticsearch.test.integration.get; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; diff --git a/src/test/java/org/elasticsearch/test/integration/document/MoreLikeThisActionTests.java b/src/test/java/org/elasticsearch/test/integration/mlt/MoreLikeThisActionTests.java similarity index 99% rename from src/test/java/org/elasticsearch/test/integration/document/MoreLikeThisActionTests.java rename to src/test/java/org/elasticsearch/test/integration/mlt/MoreLikeThisActionTests.java index 64950fd5c5b..1204c3a1439 100644 --- a/src/test/java/org/elasticsearch/test/integration/document/MoreLikeThisActionTests.java +++ b/src/test/java/org/elasticsearch/test/integration/mlt/MoreLikeThisActionTests.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.test.integration.document; +package org.elasticsearch.test.integration.mlt; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; From beb5986256f832ef9fc4cd3639120e2b97701296 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 3 Jan 2012 01:04:08 +0200 Subject: [PATCH 100/270] nicer main action names --- .../action/TransportActions.java | 43 +++++++------------ .../action/bulk/TransportShardBulkAction.java | 3 +- .../action/count/TransportCountAction.java | 2 +- .../TransportIndexDeleteByQueryAction.java | 3 +- .../TransportShardDeleteByQueryAction.java | 3 +- .../action/get/TransportGetAction.java | 4 +- .../get/TransportShardMultiGetAction.java | 5 ++- .../percolate/TransportPercolateAction.java | 2 +- 8 files changed, 27 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/TransportActions.java b/src/main/java/org/elasticsearch/action/TransportActions.java index 9e6c061e518..f98af2c008a 100644 --- a/src/main/java/org/elasticsearch/action/TransportActions.java +++ b/src/main/java/org/elasticsearch/action/TransportActions.java @@ -24,39 +24,26 @@ package org.elasticsearch.action; */ public class TransportActions { - public static final String BULK = "indices/bulk"; - - public static final String INDEX = "indices/index/shard/index"; - + public static final String BULK = "bulk"; + public static final String INDEX = "index"; public static final String UPDATE = "update"; - - public static final String COUNT = "indices/count"; - - public static final String DELETE = "indices/index/shard/delete"; - - public static final String DELETE_BY_QUERY = "indices/deleteByQuery"; - - public static final String GET = "indices/get"; - - public static final String MULTI_GET = "indices/mget"; - - public static final String SEARCH = "indices/search"; - - public static final String SEARCH_SCROLL = "indices/searchScroll"; - - public static final String TERMS = "indices/terms"; - - public static final String MORE_LIKE_THIS = "indices/moreLikeThis"; - - public static final String PERCOLATE = "indices/percolate"; + public static final String COUNT = "count"; + public static final String DELETE = "delete"; + public static final String DELETE_BY_QUERY = "deleteByQuery"; + public static final String GET = "get"; + public static final String MULTI_GET = "mget"; + public static final String SEARCH = "search"; + public static final String SEARCH_SCROLL = "searchScroll"; + public static final String MORE_LIKE_THIS = "mlt"; + public static final String PERCOLATE = "percolate"; public static class Admin { public static class Indices { - public static final String CREATE = "indices/createIndex"; - public static final String DELETE = "indices/deleteIndex"; - public static final String OPEN = "indices/openIndex"; - public static final String CLOSE = "indices/closeIndex"; + public static final String CREATE = "indices/create"; + public static final String DELETE = "indices/delete"; + public static final String OPEN = "indices/open"; + public static final String CLOSE = "indices/close"; public static final String FLUSH = "indices/flush"; public static final String REFRESH = "indices/refresh"; public static final String OPTIMIZE = "indices/optimize"; diff --git a/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index dbd0ed0df8b..d397be17531 100644 --- a/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.RoutingMissingException; +import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexRequest; @@ -102,7 +103,7 @@ public class TransportShardBulkAction extends TransportShardReplicationOperation @Override protected String transportAction() { - return "indices/index/shard/bulk"; + return TransportActions.BULK + "/shard"; } @Override diff --git a/src/main/java/org/elasticsearch/action/count/TransportCountAction.java b/src/main/java/org/elasticsearch/action/count/TransportCountAction.java index 117730cae4c..5f59c4ced22 100644 --- a/src/main/java/org/elasticsearch/action/count/TransportCountAction.java +++ b/src/main/java/org/elasticsearch/action/count/TransportCountAction.java @@ -69,7 +69,7 @@ public class TransportCountAction extends TransportBroadcastOperationAction { @@ -70,7 +68,7 @@ public class TransportGetAction extends TransportShardSingleOperationAction Date: Tue, 3 Jan 2012 01:05:08 +0200 Subject: [PATCH 101/270] nicer exception names --- .../TransportShardReplicationOperationAction.java | 4 ++-- .../action/update/TransportUpdateAction.java | 8 ++++---- ...Exception.java => DocumentAlreadyExistsException.java} | 4 ++-- ...EngineException.java => DocumentMissingException.java} | 4 ++-- ...Exception.java => DocumentSourceMissingException.java} | 4 ++-- .../org/elasticsearch/index/engine/robin/RobinEngine.java | 4 ++-- .../test/integration/document/DocumentActionsTests.java | 4 ++-- .../test/unit/index/engine/AbstractSimpleEngineTests.java | 5 ++--- 8 files changed, 18 insertions(+), 19 deletions(-) rename src/main/java/org/elasticsearch/index/engine/{DocumentAlreadyExistsEngineException.java => DocumentAlreadyExistsException.java} (87%) rename src/main/java/org/elasticsearch/index/engine/{DocumentMIssingEngineException.java => DocumentMissingException.java} (87%) rename src/main/java/org/elasticsearch/index/engine/{DocumentSourceMissingEngineException.java => DocumentSourceMissingException.java} (87%) diff --git a/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java b/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java index 1aa6aa36f7c..1524612dc75 100644 --- a/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java @@ -39,7 +39,7 @@ import org.elasticsearch.common.io.stream.VoidStreamable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.IndexShardMissingException; -import org.elasticsearch.index.engine.DocumentAlreadyExistsEngineException; +import org.elasticsearch.index.engine.DocumentAlreadyExistsException; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.shard.IllegalIndexShardStateException; import org.elasticsearch.index.shard.ShardId; @@ -178,7 +178,7 @@ public abstract class TransportShardReplicationOperationAction Date: Tue, 3 Jan 2012 01:11:47 +0200 Subject: [PATCH 102/270] document missing should be 404 --- .../elasticsearch/index/engine/DocumentMissingException.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/index/engine/DocumentMissingException.java b/src/main/java/org/elasticsearch/index/engine/DocumentMissingException.java index 02237e4141e..2603f15860c 100644 --- a/src/main/java/org/elasticsearch/index/engine/DocumentMissingException.java +++ b/src/main/java/org/elasticsearch/index/engine/DocumentMissingException.java @@ -33,6 +33,6 @@ public class DocumentMissingException extends EngineException { @Override public RestStatus status() { - return RestStatus.CONFLICT; + return RestStatus.NOT_FOUND; } } From 8e2cd64ee37d3079a416e9a7d065f7a71afac83f Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 3 Jan 2012 13:04:18 +0200 Subject: [PATCH 103/270] remove unused script service --- .../action/get/TransportShardMultiGetAction.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java index 29921f1ac9b..1121eb019e1 100644 --- a/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java +++ b/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java @@ -35,7 +35,6 @@ import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.service.IndexService; import org.elasticsearch.index.shard.service.IndexShard; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -43,16 +42,13 @@ public class TransportShardMultiGetAction extends TransportShardSingleOperationA private final IndicesService indicesService; - private final ScriptService scriptService; - private final boolean realtime; @Inject public TransportShardMultiGetAction(Settings settings, ClusterService clusterService, TransportService transportService, - IndicesService indicesService, ScriptService scriptService, ThreadPool threadPool) { + IndicesService indicesService, ThreadPool threadPool) { super(settings, threadPool, clusterService, transportService); this.indicesService = indicesService; - this.scriptService = scriptService; this.realtime = settings.getAsBoolean("action.get.realtime", true); } From a793336635b8df4f4d167159d3435cea2a5a8222 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 3 Jan 2012 13:23:02 +0200 Subject: [PATCH 104/270] clean unused method --- .../instance/TransportInstanceSingleOperationAction.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java index 406935e719c..73862aada89 100644 --- a/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java @@ -83,10 +83,6 @@ public abstract class TransportInstanceSingleOperationAction Date: Tue, 3 Jan 2012 13:43:27 +0200 Subject: [PATCH 105/270] simplify support transport actions, no need to provide several action paths, we can derive them --- .../admin/cluster/node/info/TransportNodesInfoAction.java | 5 ----- .../cluster/node/restart/TransportNodesRestartAction.java | 5 ----- .../cluster/node/stats/TransportNodesStatsAction.java | 5 ----- .../ping/broadcast/TransportBroadcastPingAction.java | 5 ----- .../cluster/ping/single/TransportSinglePingAction.java | 5 ----- .../admin/indices/analyze/TransportAnalyzeAction.java | 5 ----- .../cache/clear/TransportClearIndicesCacheAction.java | 8 -------- .../action/admin/indices/flush/TransportFlushAction.java | 7 ------- .../gateway/snapshot/TransportGatewaySnapshotAction.java | 5 ----- .../admin/indices/optimize/TransportOptimizeAction.java | 7 ------- .../admin/indices/refresh/TransportRefreshAction.java | 5 ----- .../indices/segments/TransportIndicesSegmentsAction.java | 5 ----- .../admin/indices/stats/TransportIndicesStatsAction.java | 5 ----- .../indices/status/TransportIndicesStatusAction.java | 5 ----- .../validate/query/TransportValidateQueryAction.java | 5 ----- .../elasticsearch/action/count/TransportCountAction.java | 5 ----- .../org/elasticsearch/action/get/TransportGetAction.java | 5 ----- .../action/get/TransportShardMultiGetAction.java | 5 ----- .../action/percolate/TransportPercolateAction.java | 5 ----- .../broadcast/TransportBroadcastOperationAction.java | 4 +--- .../support/nodes/TransportNodesOperationAction.java | 4 +--- .../custom/TransportSingleCustomOperationAction.java | 4 +--- .../single/shard/TransportShardSingleOperationAction.java | 4 +--- .../gateway/local/TransportNodesListGatewayMetaState.java | 5 ----- .../local/TransportNodesListGatewayStartedShards.java | 5 ----- .../store/TransportNodesListShardStoreMetaData.java | 5 ----- 26 files changed, 4 insertions(+), 129 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java index 83a0753cc24..80d335ad745 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java @@ -60,11 +60,6 @@ public class TransportNodesInfoAction extends TransportNodesOperationAction nodesInfos = new ArrayList(); diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/restart/TransportNodesRestartAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/restart/TransportNodesRestartAction.java index aecc80f6a4b..8c8cfcc1bf0 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/restart/TransportNodesRestartAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/restart/TransportNodesRestartAction.java @@ -79,11 +79,6 @@ public class TransportNodesRestartAction extends TransportNodesOperationAction nodeRestartResponses = newArrayList(); diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index c60827f9f10..6a16c298404 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -60,11 +60,6 @@ public class TransportNodesStatsAction extends TransportNodesOperationAction nodeStats = Lists.newArrayList(); diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/TransportBroadcastPingAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/TransportBroadcastPingAction.java index ba253e0a88c..ec3eb79e38b 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/TransportBroadcastPingAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/TransportBroadcastPingAction.java @@ -59,11 +59,6 @@ public class TransportBroadcastPingAction extends TransportBroadcastOperationAct return TransportActions.Admin.Cluster.Ping.BROADCAST; } - @Override - protected String transportShardAction() { - return "/cluster/ping/broadcast/shard"; - } - @Override protected BroadcastPingRequest newRequest() { return new BroadcastPingRequest(); diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/TransportSinglePingAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/TransportSinglePingAction.java index 2503f9b14b4..0c5191dbecd 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/TransportSinglePingAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/TransportSinglePingAction.java @@ -50,11 +50,6 @@ public class TransportSinglePingAction extends TransportShardSingleOperationActi return TransportActions.Admin.Cluster.Ping.SINGLE; } - @Override - protected String transportShardAction() { - return "/cluster/ping/single/shard"; - } - @Override protected ShardIterator shards(ClusterState clusterState, SinglePingRequest request) throws ElasticSearchException { return clusterService.operationRouting() diff --git a/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index ecd4c68fdb3..aa93fb76c07 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -86,11 +86,6 @@ public class TransportAnalyzeAction extends TransportSingleCustomOperationAction return TransportActions.Admin.Indices.ANALYZE; } - @Override - protected String transportShardAction() { - return "indices/analyze/shard"; - } - @Override protected ShardsIterator shards(ClusterState clusterState, AnalyzeRequest request) { if (request.index() == null) { diff --git a/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java b/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java index db563c26621..dcf4219eb48 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java @@ -43,8 +43,6 @@ import static com.google.common.collect.Lists.newArrayList; /** * Indices clear cache action. - * - * */ public class TransportClearIndicesCacheAction extends TransportBroadcastOperationAction { @@ -67,12 +65,6 @@ public class TransportClearIndicesCacheAction extends TransportBroadcastOperatio return TransportActions.Admin.Indices.Cache.CLEAR; } - @Override - protected String transportShardAction() { - return "indices/cache/clear/shard"; - } - - @Override protected ClearIndicesCacheRequest newRequest() { return new ClearIndicesCacheRequest(); diff --git a/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java b/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java index ac5d7dc093d..45269e8a3e4 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java @@ -44,8 +44,6 @@ import static com.google.common.collect.Lists.newArrayList; /** * Flush Action. - * - * */ public class TransportFlushAction extends TransportBroadcastOperationAction { @@ -67,11 +65,6 @@ public class TransportFlushAction extends TransportBroadcastOperationAction { @@ -70,11 +68,6 @@ public class TransportOptimizeAction extends TransportBroadcastOperationAction Date: Tue, 3 Jan 2012 14:03:27 +0200 Subject: [PATCH 106/270] Plugins: If a plugin has a bin directory, move it under the main bin location under the plugin name, closes #1584. --- .../org/elasticsearch/plugins/PluginManager.java | 12 ++++++++++++ .../org/elasticsearch/plugins/PluginsService.java | 10 +++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/plugins/PluginManager.java b/src/main/java/org/elasticsearch/plugins/PluginManager.java index 89cf71e049b..5cd734e37d9 100644 --- a/src/main/java/org/elasticsearch/plugins/PluginManager.java +++ b/src/main/java/org/elasticsearch/plugins/PluginManager.java @@ -208,6 +208,14 @@ public class PluginManager { pluginFile.delete(); } + File binFile = new File(extractLocation, "bin"); + if (binFile.exists() && binFile.isDirectory()) { + File toLocation = new File(new File(environment.homeFile(), "bin"), name); + System.out.println("Found bin, moving to " + toLocation.getAbsolutePath()); + FileSystemUtils.deleteRecursively(toLocation); + binFile.renameTo(toLocation); + } + // try and identify the plugin type, see if it has no .class or .jar files in it // so its probably a _site, and it it does not have a _site in it, move everything to _site if (!new File(extractLocation, "_site").exists()) { @@ -233,6 +241,10 @@ public class PluginManager { if (pluginToDelete.exists()) { pluginToDelete.delete(); } + File binLocation = new File(new File(environment.homeFile(), "bin"), name); + if (binLocation.exists()) { + FileSystemUtils.deleteRecursively(binLocation); + } } public static void main(String[] args) { diff --git a/src/main/java/org/elasticsearch/plugins/PluginsService.java b/src/main/java/org/elasticsearch/plugins/PluginsService.java index b82d89112cb..fe699eabc5f 100644 --- a/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -192,8 +192,16 @@ public class PluginsService extends AbstractComponent { try { // add the root addURL.invoke(classLoader, pluginFile.toURI().toURL()); + // gather files to add + List libFiles = Lists.newArrayList(); + libFiles.addAll(Arrays.asList(pluginsFile.listFiles())); + File libLocation = new File(pluginFile, "lib"); + if (libLocation.exists() && libLocation.isDirectory()) { + libFiles.addAll(Arrays.asList(libLocation.listFiles())); + } + // if there are jars in it, add it as well - for (File jarToAdd : pluginFile.listFiles()) { + for (File jarToAdd : libFiles) { if (!(jarToAdd.getName().endsWith(".jar") || jarToAdd.getName().endsWith(".zip"))) { continue; } From 700514fd6c900d02d83a459b2d295c57c7962123 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 3 Jan 2012 20:35:42 +0200 Subject: [PATCH 107/270] have the zip bat windows file use dos line endings --- src/main/assemblies/zip-bin.xml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/assemblies/zip-bin.xml b/src/main/assemblies/zip-bin.xml index 506b0afb7ba..acd9ba77322 100644 --- a/src/main/assemblies/zip-bin.xml +++ b/src/main/assemblies/zip-bin.xml @@ -11,8 +11,22 @@ bin bin + dos - * + elasticsearch.bat + plugin.bat + + + + bin + bin + 0755 + 0755 + unix + + elasticsearch.in.sh + elasticsearch + plugin From 5fa8dad4bf72c8943967e9aa5e77494baad552d0 Mon Sep 17 00:00:00 2001 From: Damien Hardy Date: Wed, 4 Jan 2012 12:30:12 +0100 Subject: [PATCH 108/270] No need for plenty of RAM for plugin utility (RAM is often already taken by the elasticsearch service running beside) --- bin/plugin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/plugin b/bin/plugin index 69192a9e6d3..6d54f2cecc0 100644 --- a/bin/plugin +++ b/bin/plugin @@ -28,4 +28,4 @@ else JAVA=`which java` fi -exec $JAVA -Delasticsearch -Des.path.home="$ES_HOME" -cp "$ES_HOME/lib/*" org.elasticsearch.plugins.PluginManager $* +exec $JAVA -Xmx64m -Xms16m -Delasticsearch -Des.path.home="$ES_HOME" -cp "$ES_HOME/lib/*" org.elasticsearch.plugins.PluginManager $* From a8af4e962b4001fb25b378bec8b498f6cd6bbf0f Mon Sep 17 00:00:00 2001 From: Damien Hardy Date: Wed, 4 Jan 2012 17:10:37 +0100 Subject: [PATCH 109/270] limit memory for plugin utility --- bin/plugin.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/plugin.bat b/bin/plugin.bat index 66216f21b7f..8107ee8e943 100644 --- a/bin/plugin.bat +++ b/bin/plugin.bat @@ -8,7 +8,7 @@ set SCRIPT_DIR=%~dp0 for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI -"%JAVA_HOME%\bin\java" -Des.path.home="%ES_HOME%" -cp "%ES_HOME%/lib/*" "org.elasticsearch.plugins.PluginManager" %* +"%JAVA_HOME%\bin\java" -Xmx64m -Xms16m -Des.path.home="%ES_HOME%" -cp "%ES_HOME%/lib/*" "org.elasticsearch.plugins.PluginManager" %* goto finally @@ -19,4 +19,4 @@ pause :finally -ENDLOCAL \ No newline at end of file +ENDLOCAL From 1b68f99449c5723a86d722beeec2276351d77fa9 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 4 Jan 2012 20:37:13 +0200 Subject: [PATCH 110/270] TTL does not respect routing when expiring (deleting) documents, closes #1586 --- .../selector/UidAndRoutingFieldSelector.java | 55 +++++++++++++++++++ .../indices/ttl/IndicesTTLService.java | 14 +++-- 2 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/elasticsearch/index/mapper/selector/UidAndRoutingFieldSelector.java diff --git a/src/main/java/org/elasticsearch/index/mapper/selector/UidAndRoutingFieldSelector.java b/src/main/java/org/elasticsearch/index/mapper/selector/UidAndRoutingFieldSelector.java new file mode 100644 index 00000000000..4fd3395d868 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/mapper/selector/UidAndRoutingFieldSelector.java @@ -0,0 +1,55 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.selector; + +import org.apache.lucene.document.FieldSelectorResult; +import org.elasticsearch.common.lucene.document.ResetFieldSelector; +import org.elasticsearch.index.mapper.internal.RoutingFieldMapper; +import org.elasticsearch.index.mapper.internal.UidFieldMapper; + +/** + * An optimized field selector that loads just the uid and the routing. + */ +public class UidAndRoutingFieldSelector implements ResetFieldSelector { + + private int match = 0; + + @Override + public FieldSelectorResult accept(String fieldName) { + if (UidFieldMapper.NAME.equals(fieldName)) { + if (++match == 2) { + return FieldSelectorResult.LOAD_AND_BREAK; + } + return FieldSelectorResult.LOAD; + } + if (RoutingFieldMapper.NAME.equals(fieldName)) { + if (++match == 2) { + return FieldSelectorResult.LOAD_AND_BREAK; + } + return FieldSelectorResult.LOAD; + } + return FieldSelectorResult.NO_LOAD; + } + + @Override + public void reset() { + match = 0; + } +} diff --git a/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java b/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java index 07be34a3f9a..d4a2b518caa 100644 --- a/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java +++ b/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java @@ -42,9 +42,10 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldMappers; import org.elasticsearch.index.mapper.Uid; +import org.elasticsearch.index.mapper.internal.RoutingFieldMapper; import org.elasticsearch.index.mapper.internal.TTLFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; -import org.elasticsearch.index.mapper.selector.UidFieldSelector; +import org.elasticsearch.index.mapper.selector.UidAndRoutingFieldSelector; import org.elasticsearch.index.service.IndexService; import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.index.shard.service.IndexShard; @@ -173,7 +174,7 @@ public class IndicesTTLService extends AbstractLifecycleComponent docsToPurge = expiredDocsCollector.getDocsToPurge(); BulkRequestBuilder bulkRequest = client.prepareBulk(); for (DocToPurge docToPurge : docsToPurge) { - bulkRequest.add(new DeleteRequest().index(shardToPurge.routingEntry().index()).type(docToPurge.type).id(docToPurge.id).version(docToPurge.version)); + bulkRequest.add(new DeleteRequest().index(shardToPurge.routingEntry().index()).type(docToPurge.type).id(docToPurge.id).version(docToPurge.version).routing(docToPurge.routing)); bulkRequest = processBulkIfNeeded(bulkRequest, false); } processBulkIfNeeded(bulkRequest, true); @@ -189,11 +190,13 @@ public class IndicesTTLService extends AbstractLifecycleComponent Date: Wed, 4 Jan 2012 20:44:39 +0200 Subject: [PATCH 111/270] move to 0.9.2 lzf --- .../common/compress/lzf/ChunkDecoder.java | 2 +- .../common/compress/lzf/LZFDecoder.java | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/elasticsearch/common/compress/lzf/ChunkDecoder.java b/src/main/java/org/elasticsearch/common/compress/lzf/ChunkDecoder.java index fd3d5427f15..1b787361949 100755 --- a/src/main/java/org/elasticsearch/common/compress/lzf/ChunkDecoder.java +++ b/src/main/java/org/elasticsearch/common/compress/lzf/ChunkDecoder.java @@ -163,7 +163,7 @@ public abstract class ChunkDecoder { ++blockNr; } // one more sanity check: - if (ptr != data.length) { + if (ptr != end) { throw new IOException("Corrupt input data: block #" + blockNr + " extends " + (data.length - ptr) + " beyond end of input"); } return uncompressedSize; diff --git a/src/main/java/org/elasticsearch/common/compress/lzf/LZFDecoder.java b/src/main/java/org/elasticsearch/common/compress/lzf/LZFDecoder.java index 0028ef20f93..784137e053a 100644 --- a/src/main/java/org/elasticsearch/common/compress/lzf/LZFDecoder.java +++ b/src/main/java/org/elasticsearch/common/compress/lzf/LZFDecoder.java @@ -37,19 +37,19 @@ public class LZFDecoder { return decode(inputBuffer, 0, inputBuffer.length); } - public static byte[] decode(final byte[] inputBuffer, int inputPtr, int inputLen) throws IOException { - return ChunkDecoderFactory.optimalInstance().decode(inputBuffer); + public static byte[] decode(final byte[] inputBuffer, int offset, int length) throws IOException { + return ChunkDecoderFactory.optimalInstance().decode(inputBuffer, offset, length); } public static int decode(final byte[] inputBuffer, final byte[] targetBuffer) throws IOException { return decode(inputBuffer, 0, inputBuffer.length, targetBuffer); } - public static int decode(final byte[] sourceBuffer, int inPtr, int inLength, final byte[] targetBuffer) throws IOException { - return ChunkDecoderFactory.optimalInstance().decode(sourceBuffer, inPtr, inLength, targetBuffer); + public static int decode(final byte[] sourceBuffer, int offset, int length, final byte[] targetBuffer) throws IOException { + return ChunkDecoderFactory.optimalInstance().decode(sourceBuffer, offset, length, targetBuffer); } - public static int calculateUncompressedSize(byte[] data, int ptr, int length) throws IOException { - return ChunkDecoder.calculateUncompressedSize(data, ptr, length); + public static int calculateUncompressedSize(byte[] data, int offset, int length) throws IOException { + return ChunkDecoder.calculateUncompressedSize(data, offset, length); } } From e5f2ce0fd644412298b5ef35ebcdcc074b23795c Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 4 Jan 2012 21:53:26 +0200 Subject: [PATCH 112/270] use factor in scripts, so custom score function will work correctly when it multiplies --- .../search/function/BoostScoreFunction.java | 17 ++++++- .../function/FiltersFunctionScoreQuery.java | 51 ++++++++++--------- .../search/function/FunctionScoreQuery.java | 4 +- .../lucene/search/function/ScoreFunction.java | 6 ++- .../index/query/CustomScoreQueryParser.java | 14 ++++- .../customscore/CustomScoreSearchTests.java | 4 +- 6 files changed, 63 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/elasticsearch/common/lucene/search/function/BoostScoreFunction.java b/src/main/java/org/elasticsearch/common/lucene/search/function/BoostScoreFunction.java index d4fd30c3cf4..99b2bcbdbed 100644 --- a/src/main/java/org/elasticsearch/common/lucene/search/function/BoostScoreFunction.java +++ b/src/main/java/org/elasticsearch/common/lucene/search/function/BoostScoreFunction.java @@ -49,13 +49,23 @@ public class BoostScoreFunction implements ScoreFunction { } @Override - public Explanation explain(int docId, Explanation subQueryExpl) { + public float factor(int docId) { + return boost; + } + + @Override + public Explanation explainScore(int docId, Explanation subQueryExpl) { Explanation exp = new Explanation(boost * subQueryExpl.getValue(), "static boost function: product of:"); exp.addDetail(subQueryExpl); exp.addDetail(new Explanation(boost, "boostFactor")); return exp; } + @Override + public Explanation explainFactor(int docId) { + return new Explanation(boost, "boostFactor"); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -72,4 +82,9 @@ public class BoostScoreFunction implements ScoreFunction { public int hashCode() { return (boost != +0.0f ? Float.floatToIntBits(boost) : 0); } + + @Override + public String toString() { + return "boost[" + boost + "]"; + } } diff --git a/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java b/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java index 2c682f556c0..43263a466a7 100644 --- a/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java +++ b/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java @@ -34,8 +34,6 @@ import java.util.Set; /** * A query that allows for a pluggable boost function / filter. If it matches the filter, it will * be boosted by the formula. - * - * */ public class FiltersFunctionScoreQuery extends Query { @@ -166,7 +164,7 @@ public class FiltersFunctionScoreQuery extends Query { DocSet docSet = DocSets.convert(reader, filterFunction.filter.getDocIdSet(reader)); if (docSet.get(doc)) { filterFunction.function.setNextReader(reader); - Explanation functionExplanation = filterFunction.function.explain(doc, subQueryExpl); + Explanation functionExplanation = filterFunction.function.explainFactor(doc); float sc = getValue() * functionExplanation.getValue(); Explanation res = new ComplexExplanation(true, sc, "custom score, product of:"); res.addDetail(new Explanation(1.0f, "match filter: " + filterFunction.filter.toString())); @@ -186,7 +184,7 @@ public class FiltersFunctionScoreQuery extends Query { DocSet docSet = DocSets.convert(reader, filterFunction.filter.getDocIdSet(reader)); if (docSet.get(doc)) { filterFunction.function.setNextReader(reader); - Explanation functionExplanation = filterFunction.function.explain(doc, subQueryExpl); + Explanation functionExplanation = filterFunction.function.explainFactor(doc); float sc = functionExplanation.getValue(); count++; total += sc; @@ -221,6 +219,7 @@ public class FiltersFunctionScoreQuery extends Query { } sc *= getValue(); Explanation res = new ComplexExplanation(true, sc, "custom score, score mode [" + scoreMode.toString().toLowerCase() + "]"); + res.addDetail(subQueryExpl); for (Explanation explanation : filtersExplanations) { res.addDetail(explanation); } @@ -272,56 +271,58 @@ public class FiltersFunctionScoreQuery extends Query { @Override public float score() throws IOException { int docId = scorer.docID(); - float score = scorer.score(); + float factor = 1.0f; if (scoreMode == ScoreMode.First) { for (int i = 0; i < filterFunctions.length; i++) { if (docSets[i].get(docId)) { - return subQueryWeight * filterFunctions[i].function.score(docId, score); + factor = filterFunctions[i].function.factor(docId); + break; } } } else if (scoreMode == ScoreMode.Max) { - float maxScore = Float.NEGATIVE_INFINITY; + float maxFactor = Float.NEGATIVE_INFINITY; for (int i = 0; i < filterFunctions.length; i++) { if (docSets[i].get(docId)) { - maxScore = Math.max(filterFunctions[i].function.score(docId, score), maxScore); + maxFactor = Math.max(filterFunctions[i].function.factor(docId), maxFactor); } } - if (maxScore != Float.NEGATIVE_INFINITY) { - score = maxScore; + if (maxFactor != Float.NEGATIVE_INFINITY) { + factor = maxFactor; } } else if (scoreMode == ScoreMode.Min) { - float minScore = Float.POSITIVE_INFINITY; + float minFactor = Float.POSITIVE_INFINITY; for (int i = 0; i < filterFunctions.length; i++) { if (docSets[i].get(docId)) { - minScore = Math.min(filterFunctions[i].function.score(docId, score), minScore); + minFactor = Math.min(filterFunctions[i].function.factor(docId), minFactor); } } - if (minScore != Float.POSITIVE_INFINITY) { - score = minScore; + if (minFactor != Float.POSITIVE_INFINITY) { + factor = minFactor; + } + } else if (scoreMode == ScoreMode.Multiply) { + for (int i = 0; i < filterFunctions.length; i++) { + if (docSets[i].get(docId)) { + factor *= filterFunctions[i].function.factor(docId); + } } } else { // Avg / Total - float totalScore = 0.0f; - float multiplicativeScore = 1.0f; + float totalFactor = 0.0f; int count = 0; for (int i = 0; i < filterFunctions.length; i++) { if (docSets[i].get(docId)) { - float tempScore = filterFunctions[i].function.score(docId, score); - totalScore += tempScore; - multiplicativeScore *= tempScore; + totalFactor += filterFunctions[i].function.factor(docId); count++; } } if (count != 0) { - score = totalScore; + factor = totalFactor; if (scoreMode == ScoreMode.Avg) { - score /= count; - } - else if (scoreMode == ScoreMode.Multiply) { - score = multiplicativeScore; + factor /= count; } } } - return subQueryWeight * score; + float score = scorer.score(); + return subQueryWeight * score * factor; } } diff --git a/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java index 11e2e055ef4..02b59404899 100644 --- a/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java +++ b/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java @@ -29,8 +29,6 @@ import java.util.Set; /** * A query that allows for a pluggable boost function to be applied to it. - * - * */ public class FunctionScoreQuery extends Query { @@ -117,7 +115,7 @@ public class FunctionScoreQuery extends Query { } function.setNextReader(reader); - Explanation functionExplanation = function.explain(doc, subQueryExpl); + Explanation functionExplanation = function.explainScore(doc, subQueryExpl); float sc = getValue() * functionExplanation.getValue(); Explanation res = new ComplexExplanation(true, sc, "custom score, product of:"); res.addDetail(functionExplanation); diff --git a/src/main/java/org/elasticsearch/common/lucene/search/function/ScoreFunction.java b/src/main/java/org/elasticsearch/common/lucene/search/function/ScoreFunction.java index f5dd7437766..b54f140fbfa 100644 --- a/src/main/java/org/elasticsearch/common/lucene/search/function/ScoreFunction.java +++ b/src/main/java/org/elasticsearch/common/lucene/search/function/ScoreFunction.java @@ -31,5 +31,9 @@ public interface ScoreFunction { float score(int docId, float subQueryScore); - Explanation explain(int docId, Explanation subQueryExpl); + float factor(int docId); + + Explanation explainScore(int docId, Explanation subQueryExpl); + + Explanation explainFactor(int docId); } diff --git a/src/main/java/org/elasticsearch/index/query/CustomScoreQueryParser.java b/src/main/java/org/elasticsearch/index/query/CustomScoreQueryParser.java index 3741c71afe8..e866f9e8615 100644 --- a/src/main/java/org/elasticsearch/index/query/CustomScoreQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/CustomScoreQueryParser.java @@ -125,13 +125,25 @@ public class CustomScoreQueryParser implements QueryParser { } @Override - public Explanation explain(int docId, Explanation subQueryExpl) { + public float factor(int docId) { + // just the factor, so don't provide _score + script.setNextDocId(docId); + return script.runAsFloat(); + } + + @Override + public Explanation explainScore(int docId, Explanation subQueryExpl) { float score = score(docId, subQueryExpl.getValue()); Explanation exp = new Explanation(score, "script score function: product of:"); exp.addDetail(subQueryExpl); return exp; } + @Override + public Explanation explainFactor(int docId) { + return new Explanation(factor(docId), "scriptFactor"); + } + @Override public String toString() { return "script[" + sScript + "], params [" + params + "]"; diff --git a/src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.java index 33b9a4b840d..ea9239a8b96 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/customscore/CustomScoreSearchTests.java @@ -164,8 +164,8 @@ public class CustomScoreSearchTests extends AbstractNodesTests { SearchResponse searchResponse = client.prepareSearch("test") .setQuery(customFiltersScoreQuery(matchAllQuery()) - .add(termFilter("field", "value4"), "_score * 2") - .add(termFilter("field", "value2"), "_score * 3")) + .add(termFilter("field", "value4"), "2") + .add(termFilter("field", "value2"), "3")) .setExplain(true) .execute().actionGet(); From df4a3bfd379fd90b3667bbcd7ca2ed4619fee025 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 5 Jan 2012 00:09:40 +0200 Subject: [PATCH 113/270] better explanation calc with the change to factor in custom filters score --- .../function/FiltersFunctionScoreQuery.java | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java b/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java index 43263a466a7..6a8d0c8b68a 100644 --- a/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java +++ b/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java @@ -165,7 +165,7 @@ public class FiltersFunctionScoreQuery extends Query { if (docSet.get(doc)) { filterFunction.function.setNextReader(reader); Explanation functionExplanation = filterFunction.function.explainFactor(doc); - float sc = getValue() * functionExplanation.getValue(); + float sc = getValue() * subQueryExpl.getValue() * functionExplanation.getValue(); Explanation res = new ComplexExplanation(true, sc, "custom score, product of:"); res.addDetail(new Explanation(1.0f, "match filter: " + filterFunction.filter.toString())); res.addDetail(functionExplanation); @@ -185,13 +185,13 @@ public class FiltersFunctionScoreQuery extends Query { if (docSet.get(doc)) { filterFunction.function.setNextReader(reader); Explanation functionExplanation = filterFunction.function.explainFactor(doc); - float sc = functionExplanation.getValue(); + float factor = functionExplanation.getValue(); count++; - total += sc; - multiply *= sc; - max = Math.max(sc, max); - min = Math.min(sc, min); - Explanation res = new ComplexExplanation(true, sc, "custom score, product of:"); + total += factor; + multiply *= factor; + max = Math.max(factor, max); + min = Math.min(factor, min); + Explanation res = new ComplexExplanation(true, factor, "custom score, product of:"); res.addDetail(new Explanation(1.0f, "match filter: " + filterFunction.filter.toString())); res.addDetail(functionExplanation); res.addDetail(new Explanation(getValue(), "queryBoost")); @@ -199,25 +199,26 @@ public class FiltersFunctionScoreQuery extends Query { } } if (count > 0) { - float sc = 0; + float factor = 0; switch (scoreMode) { case Avg: - sc = total / count; + factor = total / count; break; case Max: - sc = max; + factor = max; break; case Min: - sc = min; + factor = min; break; case Total: - sc = total; + factor = total; break; case Multiply: - sc = multiply; + factor = multiply; break; } - sc *= getValue(); + + float sc = factor * subQueryExpl.getValue() * getValue(); Explanation res = new ComplexExplanation(true, sc, "custom score, score mode [" + scoreMode.toString().toLowerCase() + "]"); res.addDetail(subQueryExpl); for (Explanation explanation : filtersExplanations) { From d95aa9f266ccecf33894c88af329895c1693cd14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20Deve=CC=80ze?= Date: Wed, 4 Jan 2012 23:37:34 +0100 Subject: [PATCH 114/270] add ttl tests with routing --- .../test/integration/ttl/SimpleTTLTests.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/elasticsearch/test/integration/ttl/SimpleTTLTests.java b/src/test/java/org/elasticsearch/test/integration/ttl/SimpleTTLTests.java index 89f4ab78fe1..3b8f00c041b 100644 --- a/src/test/java/org/elasticsearch/test/integration/ttl/SimpleTTLTests.java +++ b/src/test/java/org/elasticsearch/test/integration/ttl/SimpleTTLTests.java @@ -39,7 +39,12 @@ public class SimpleTTLTests extends AbstractNodesTests { @BeforeClass public void createNodes() throws Exception { - Settings settings = settingsBuilder().put("indices.ttl.interval", purgeInterval).build(); + Settings settings = settingsBuilder() + .put("indices.ttl.interval", purgeInterval) + .put("index.number_of_shards", 2) // 2 shards to test TTL purge with routing properly + .put("cluster.routing.operation.use_type", false) // make sure we control the shard computation + .put("cluster.routing.operation.hash.type", "djb") + .build(); startNode("node1", settings); startNode("node2", settings); client = getClient(); @@ -71,7 +76,9 @@ public class SimpleTTLTests extends AbstractNodesTests { client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); long providedTTLValue = 3000; logger.info("--> checking ttl"); + // Index one doc without routing and one doc with routing client.prepareIndex("test", "type1", "1").setSource("field1", "value1").setTTL(providedTTLValue).setRefresh(true).execute().actionGet(); + client.prepareIndex("test", "type1", "with_routing").setSource("field1", "value1").setTTL(providedTTLValue).setRouting("routing").setRefresh(true).execute().actionGet(); long now = System.currentTimeMillis(); // realtime get check @@ -100,7 +107,7 @@ public class SimpleTTLTests extends AbstractNodesTests { assertThat(ttl0, lessThan(providedTTLValue - (now1 - now))); logger.info("--> checking purger"); - // make sure the purger has done its job + // make sure the purger has done its job for all indexed docs that are expired long shouldBeExpiredDate = now + providedTTLValue + purgeInterval + 2000; now1 = System.currentTimeMillis(); if (shouldBeExpiredDate - now1 > 0) { @@ -109,14 +116,22 @@ public class SimpleTTLTests extends AbstractNodesTests { // realtime get check getResponse = client.prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(true).execute().actionGet(); assertThat(getResponse.exists(), equalTo(false)); + getResponse = client.prepareGet("test", "type1", "with_routing").setRouting("routing").setFields("_ttl").setRealtime(true).execute().actionGet(); + assertThat(getResponse.exists(), equalTo(false)); // replica realtime get check getResponse = client.prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(true).execute().actionGet(); assertThat(getResponse.exists(), equalTo(false)); + getResponse = client.prepareGet("test", "type1", "with_routing").setRouting("routing").setFields("_ttl").setRealtime(true).execute().actionGet(); + assertThat(getResponse.exists(), equalTo(false)); // non realtime get (stored) check getResponse = client.prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).execute().actionGet(); assertThat(getResponse.exists(), equalTo(false)); + getResponse = client.prepareGet("test", "type1", "with_routing").setRouting("routing").setFields("_ttl").setRealtime(false).execute().actionGet(); + assertThat(getResponse.exists(), equalTo(false)); // non realtime get going the replica check getResponse = client.prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).execute().actionGet(); assertThat(getResponse.exists(), equalTo(false)); + getResponse = client.prepareGet("test", "type1", "with_routing").setRouting("routing").setFields("_ttl").setRealtime(false).execute().actionGet(); + assertThat(getResponse.exists(), equalTo(false)); } } From a18021c77859c477d03ebc5fe7d13bf036ea0946 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 5 Jan 2012 20:44:09 +0200 Subject: [PATCH 115/270] Filter cache to have just weighted (node) and none, and index query parser cache to be size based, closes #1590. --- pom.xml | 14 +- .../common/cache/CacheBuilderHelper.java | 34 +++ .../common/inject/internal/BytecodeGen.java | 236 ------------------ .../inject/internal/ConstructionContext.java | 4 +- .../common/inject/util/Modules.java | 2 +- .../data/resident/ResidentFieldDataCache.java | 29 ++- .../field/data/soft/SoftFieldDataCache.java | 24 +- .../AbstractConcurrentMapFieldDataCache.java | 36 ++- .../field/data/weak/WeakFieldDataCache.java | 66 ----- .../index/cache/filter/FilterCacheModule.java | 4 +- .../cache/filter/node/NodeFilterCache.java | 61 ----- .../filter/resident/ResidentFilterCache.java | 129 ---------- .../cache/filter/soft/SoftFilterCache.java | 131 ---------- .../AbstractConcurrentMapFilterCache.java | 176 ------------- .../cache/filter/weak/WeakFilterCache.java | 129 ---------- .../WeightedFilterCache.java} | 97 +++---- .../query/parser/QueryParserCacheModule.java | 4 +- .../resident/ResidentQueryParserCache.java | 86 +++++++ .../parser/soft/SoftQueryParserCache.java | 40 --- .../parser/weak/WeakQueryParserCache.java | 40 --- .../elasticsearch/indices/IndicesModule.java | 4 +- .../cache/filter/IndicesFilterCache.java | 154 ++++++++++++ .../cache/filter/IndicesNodeFilterCache.java | 123 --------- .../node/internal/InternalNode.java | 4 +- .../elasticsearch/script/ScriptService.java | 10 +- .../index/cache/filter/FilterCacheTests.java | 13 - 26 files changed, 392 insertions(+), 1258 deletions(-) create mode 100644 src/main/java/org/elasticsearch/common/cache/CacheBuilderHelper.java delete mode 100644 src/main/java/org/elasticsearch/common/inject/internal/BytecodeGen.java delete mode 100644 src/main/java/org/elasticsearch/index/cache/field/data/weak/WeakFieldDataCache.java delete mode 100644 src/main/java/org/elasticsearch/index/cache/filter/node/NodeFilterCache.java delete mode 100644 src/main/java/org/elasticsearch/index/cache/filter/resident/ResidentFilterCache.java delete mode 100644 src/main/java/org/elasticsearch/index/cache/filter/soft/SoftFilterCache.java delete mode 100644 src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java delete mode 100644 src/main/java/org/elasticsearch/index/cache/filter/weak/WeakFilterCache.java rename src/main/java/org/elasticsearch/index/cache/filter/{support/AbstractWeightedFilterCache.java => weighted/WeightedFilterCache.java} (67%) create mode 100644 src/main/java/org/elasticsearch/index/cache/query/parser/resident/ResidentQueryParserCache.java delete mode 100644 src/main/java/org/elasticsearch/index/cache/query/parser/soft/SoftQueryParserCache.java delete mode 100644 src/main/java/org/elasticsearch/index/cache/query/parser/weak/WeakQueryParserCache.java create mode 100644 src/main/java/org/elasticsearch/indices/cache/filter/IndicesFilterCache.java delete mode 100644 src/main/java/org/elasticsearch/indices/cache/filter/IndicesNodeFilterCache.java diff --git a/pom.xml b/pom.xml index 694918e3ab3..8fa4d30e91e 100644 --- a/pom.xml +++ b/pom.xml @@ -81,17 +81,10 @@ - - com.googlecode.concurrentlinkedhashmap - concurrentlinkedhashmap-lru - 1.2 - compile - - com.google.guava guava - 10.0.1 + 11.0 compile @@ -315,7 +308,6 @@ true - com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru com.google.guava:guava net.sf.trove4j:trove4j org.elasticsearch:es-jsr166y @@ -330,10 +322,6 @@ - - com.googlecode.concurrentlinkedhashmap - org.elasticsearch.common.concurrentlinkedhashmap - com.google.common org.elasticsearch.common diff --git a/src/main/java/org/elasticsearch/common/cache/CacheBuilderHelper.java b/src/main/java/org/elasticsearch/common/cache/CacheBuilderHelper.java new file mode 100644 index 00000000000..01ca9d9670b --- /dev/null +++ b/src/main/java/org/elasticsearch/common/cache/CacheBuilderHelper.java @@ -0,0 +1,34 @@ +package org.elasticsearch.common.cache; + +import com.google.common.cache.CacheBuilder; + +import java.lang.reflect.Method; + +/** + */ +public class CacheBuilderHelper { + + private static final Method cacheBuilderDisableStatsMethod; + + static { + Method cacheBuilderDisableStatsMethodX = null; + try { + cacheBuilderDisableStatsMethodX = CacheBuilder.class.getDeclaredMethod("disableStats"); + cacheBuilderDisableStatsMethodX.setAccessible(true); + } catch (Exception e) { + e.printStackTrace(); + } + cacheBuilderDisableStatsMethod = cacheBuilderDisableStatsMethodX; + } + + public static void disableStats(CacheBuilder cacheBuilder) { + if (cacheBuilderDisableStatsMethod != null) { + try { + cacheBuilderDisableStatsMethod.invoke(cacheBuilder); + } catch (Exception e) { + e.printStackTrace(); + // ignore + } + } + } +} diff --git a/src/main/java/org/elasticsearch/common/inject/internal/BytecodeGen.java b/src/main/java/org/elasticsearch/common/inject/internal/BytecodeGen.java deleted file mode 100644 index 113a9ed7991..00000000000 --- a/src/main/java/org/elasticsearch/common/inject/internal/BytecodeGen.java +++ /dev/null @@ -1,236 +0,0 @@ -/** - * Copyright (C) 2008 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.elasticsearch.common.inject.internal; - -import com.google.common.base.Function; -import com.google.common.collect.MapMaker; - -import java.lang.reflect.Constructor; -import java.lang.reflect.Member; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.Map; -import java.util.logging.Logger; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * Utility methods for runtime code generation and class loading. We use this stuff for {@link - * net.sf.cglib.reflect.FastClass faster reflection}, {@link net.sf.cglib.proxy.Enhancer method - * interceptors} and to proxy circular dependencies. - *

- *

When loading classes, we need to be careful of: - *

    - *
  • Memory leaks. Generated classes need to be garbage collected in long-lived - * applications. Once an injector and any instances it created can be garbage collected, the - * corresponding generated classes should be collectable. - *
  • Visibility. Containers like OSGi use class loader boundaries - * to enforce modularity at runtime. - *
- *

- *

For each generated class, there's multiple class loaders involved: - *

    - *
  • The related class's class loader. Every generated class services exactly - * one user-supplied class. This class loader must be used to access members with private and - * package visibility. - *
  • Guice's class loader. - *
  • Our bridge class loader. This is a child of the user's class loader. It - * selectively delegates to either the user's class loader (for user classes) or the Guice - * class loader (for internal classes that are used by the generated classes). This class - * loader that owns the classes generated by Guice. - *
- * - * @author mcculls@gmail.com (Stuart McCulloch) - * @author jessewilson@google.com (Jesse Wilson) - */ -public final class BytecodeGen { - - private static final Logger logger = Logger.getLogger(BytecodeGen.class.getName()); - - static final ClassLoader GUICE_CLASS_LOADER = BytecodeGen.class.getClassLoader(); - - /** - * ie. "com.google.inject.internal" - */ - private static final String GUICE_INTERNAL_PACKAGE - = BytecodeGen.class.getName().replaceFirst("\\.internal\\..*$", ".internal"); - - private static final String CGLIB_PACKAGE = " "; // any string that's illegal in a package name - - /** - * Use "-Dguice.custom.loader=false" to disable custom classloading. - */ - static final boolean HOOK_ENABLED - = "true".equals(System.getProperty("guice.custom.loader", "true")); - - /** - * Weak cache of bridge class loaders that make the Guice implementation - * classes visible to various code-generated proxies of client classes. - */ - private static final Map CLASS_LOADER_CACHE - = new MapMaker().weakKeys().weakValues().makeComputingMap( - new Function() { - public ClassLoader apply(final @Nullable ClassLoader typeClassLoader) { - logger.fine("Creating a bridge ClassLoader for " + typeClassLoader); - return AccessController.doPrivileged(new PrivilegedAction() { - public ClassLoader run() { - return new BridgeClassLoader(typeClassLoader); - } - }); - } - }); - - /** - * For class loaders, {@code null}, is always an alias to the - * {@link ClassLoader#getSystemClassLoader() system class loader}. This method - * will not return null. - */ - private static ClassLoader canonicalize(ClassLoader classLoader) { - return classLoader != null - ? classLoader - : checkNotNull(getSystemClassLoaderOrNull(), "Couldn't get a ClassLoader"); - } - - /** - * Returns the system classloader, or {@code null} if we don't have - * permission. - */ - private static ClassLoader getSystemClassLoaderOrNull() { - try { - return ClassLoader.getSystemClassLoader(); - } catch (SecurityException e) { - return null; - } - } - - /** - * Returns the class loader to host generated classes for {@code type}. - */ - public static ClassLoader getClassLoader(Class type) { - return getClassLoader(type, type.getClassLoader()); - } - - private static ClassLoader getClassLoader(Class type, ClassLoader delegate) { - delegate = canonicalize(delegate); - - // if the application is running in the System classloader, assume we can run there too - if (delegate == getSystemClassLoaderOrNull()) { - return delegate; - } - - // Don't bother bridging existing bridge classloaders - if (delegate instanceof BridgeClassLoader) { - return delegate; - } - - if (HOOK_ENABLED && Visibility.forType(type) == Visibility.PUBLIC) { - return CLASS_LOADER_CACHE.get(delegate); - } - - return delegate; - } - - /** - * The required visibility of a user's class from a Guice-generated class. Visibility of - * package-private members depends on the loading classloader: only if two classes were loaded by - * the same classloader can they see each other's package-private members. We need to be careful - * when choosing which classloader to use for generated classes. We prefer our bridge classloader, - * since it's OSGi-safe and doesn't leak permgen space. But often we cannot due to visibility. - */ - public enum Visibility { - - /** - * Indicates that Guice-generated classes only need to call and override public members of the - * target class. These generated classes may be loaded by our bridge classloader. - */ - PUBLIC { - public Visibility and(Visibility that) { - return that; - } - }, - - /** - * Indicates that Guice-generated classes need to call or override package-private members. - * These generated classes must be loaded in the same classloader as the target class. They - * won't work with OSGi, and won't get garbage collected until the target class' classloader is - * garbage collected. - */ - SAME_PACKAGE { - public Visibility and(Visibility that) { - return this; - } - }; - - public static Visibility forMember(Member member) { - if ((member.getModifiers() & (Modifier.PROTECTED | Modifier.PUBLIC)) == 0) { - return SAME_PACKAGE; - } - - Class[] parameterTypes = member instanceof Constructor - ? ((Constructor) member).getParameterTypes() - : ((Method) member).getParameterTypes(); - for (Class type : parameterTypes) { - if (forType(type) == SAME_PACKAGE) { - return SAME_PACKAGE; - } - } - - return PUBLIC; - } - - public static Visibility forType(Class type) { - return (type.getModifiers() & (Modifier.PROTECTED | Modifier.PUBLIC)) != 0 - ? PUBLIC - : SAME_PACKAGE; - } - - public abstract Visibility and(Visibility that); - } - - /** - * Loader for Guice-generated classes. For referenced classes, this delegates to either either the - * user's classloader (which is the parent of this classloader) or Guice's class loader. - */ - private static class BridgeClassLoader extends ClassLoader { - - public BridgeClassLoader(ClassLoader usersClassLoader) { - super(usersClassLoader); - } - - @Override - protected Class loadClass(String name, boolean resolve) - throws ClassNotFoundException { - - // delegate internal requests to Guice class space - if (name.startsWith(GUICE_INTERNAL_PACKAGE) || name.startsWith(CGLIB_PACKAGE)) { - try { - Class clazz = GUICE_CLASS_LOADER.loadClass(name); - if (resolve) { - resolveClass(clazz); - } - return clazz; - } catch (Exception e) { - // fall back to classic delegation - } - } - - return super.loadClass(name, resolve); - } - } -} diff --git a/src/main/java/org/elasticsearch/common/inject/internal/ConstructionContext.java b/src/main/java/org/elasticsearch/common/inject/internal/ConstructionContext.java index fb4b0b126eb..a3397467d56 100644 --- a/src/main/java/org/elasticsearch/common/inject/internal/ConstructionContext.java +++ b/src/main/java/org/elasticsearch/common/inject/internal/ConstructionContext.java @@ -77,7 +77,9 @@ public class ConstructionContext { = new DelegatingInvocationHandler(); invocationHandlers.add(invocationHandler); - ClassLoader classLoader = BytecodeGen.getClassLoader(expectedType); + // ES: Replace, since we don't use bytecode gen, just get the type class loader, or system if its null + //ClassLoader classLoader = BytecodeGen.getClassLoader(expectedType); + ClassLoader classLoader = expectedType.getClassLoader() == null ? ClassLoader.getSystemClassLoader() : expectedType.getClassLoader(); return expectedType.cast(Proxy.newProxyInstance(classLoader, new Class[]{expectedType}, invocationHandler)); } diff --git a/src/main/java/org/elasticsearch/common/inject/util/Modules.java b/src/main/java/org/elasticsearch/common/inject/util/Modules.java index f842c6a7155..a7506cf419b 100644 --- a/src/main/java/org/elasticsearch/common/inject/util/Modules.java +++ b/src/main/java/org/elasticsearch/common/inject/util/Modules.java @@ -82,7 +82,7 @@ public final class Modules { * Returns a new module that installs all of {@code modules}. */ public static Module combine(Module... modules) { - return combine(ImmutableSet.of(modules)); + return combine(ImmutableSet.copyOf(modules)); } /** diff --git a/src/main/java/org/elasticsearch/index/cache/field/data/resident/ResidentFieldDataCache.java b/src/main/java/org/elasticsearch/index/cache/field/data/resident/ResidentFieldDataCache.java index 202abfd530d..8f8414e124c 100644 --- a/src/main/java/org/elasticsearch/index/cache/field/data/resident/ResidentFieldDataCache.java +++ b/src/main/java/org/elasticsearch/index/cache/field/data/resident/ResidentFieldDataCache.java @@ -20,11 +20,13 @@ package org.elasticsearch.index.cache.field.data.resident; import com.google.common.base.Objects; -import com.google.common.collect.MapEvictionListener; -import com.google.common.collect.MapMaker; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.cache.CacheBuilderHelper; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.settings.Settings; @@ -35,13 +37,12 @@ import org.elasticsearch.index.field.data.FieldData; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.settings.IndexSettingsService; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; /** * */ -public class ResidentFieldDataCache extends AbstractConcurrentMapFieldDataCache implements MapEvictionListener { +public class ResidentFieldDataCache extends AbstractConcurrentMapFieldDataCache implements RemovalListener { private final IndexSettingsService indexSettingsService; @@ -71,16 +72,16 @@ public class ResidentFieldDataCache extends AbstractConcurrentMapFieldDataCache } @Override - protected ConcurrentMap buildFieldDataMap() { - MapMaker mapMaker = new MapMaker(); + protected Cache buildFieldDataMap() { + CacheBuilder cacheBuilder = CacheBuilder.newBuilder().removalListener(this); if (maxSize != -1) { - mapMaker.maximumSize(maxSize); + cacheBuilder.maximumSize(maxSize); } if (expire != null) { - mapMaker.expireAfterAccess(expire.nanos(), TimeUnit.NANOSECONDS); + cacheBuilder.expireAfterAccess(expire.nanos(), TimeUnit.NANOSECONDS); } - mapMaker.evictionListener(this); - return mapMaker.makeMap(); + CacheBuilderHelper.disableStats(cacheBuilder); + return cacheBuilder.build(); } @Override @@ -94,8 +95,10 @@ public class ResidentFieldDataCache extends AbstractConcurrentMapFieldDataCache } @Override - public void onEviction(@Nullable String s, @Nullable FieldData fieldData) { - evictions.inc(); + public void onRemoval(RemovalNotification removalNotification) { + if (removalNotification.wasEvicted()) { + evictions.inc(); + } } static { diff --git a/src/main/java/org/elasticsearch/index/cache/field/data/soft/SoftFieldDataCache.java b/src/main/java/org/elasticsearch/index/cache/field/data/soft/SoftFieldDataCache.java index c49f84e5008..ffa5b01f0ef 100644 --- a/src/main/java/org/elasticsearch/index/cache/field/data/soft/SoftFieldDataCache.java +++ b/src/main/java/org/elasticsearch/index/cache/field/data/soft/SoftFieldDataCache.java @@ -19,9 +19,11 @@ package org.elasticsearch.index.cache.field.data.soft; -import com.google.common.collect.MapEvictionListener; -import com.google.common.collect.MapMaker; -import org.elasticsearch.common.Nullable; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; +import org.elasticsearch.common.cache.CacheBuilderHelper; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.settings.Settings; @@ -30,12 +32,10 @@ import org.elasticsearch.index.cache.field.data.support.AbstractConcurrentMapFie import org.elasticsearch.index.field.data.FieldData; import org.elasticsearch.index.settings.IndexSettings; -import java.util.concurrent.ConcurrentMap; - /** * */ -public class SoftFieldDataCache extends AbstractConcurrentMapFieldDataCache implements MapEvictionListener { +public class SoftFieldDataCache extends AbstractConcurrentMapFieldDataCache implements RemovalListener { private final CounterMetric evictions = new CounterMetric(); @@ -45,8 +45,10 @@ public class SoftFieldDataCache extends AbstractConcurrentMapFieldDataCache impl } @Override - protected ConcurrentMap buildFieldDataMap() { - return new MapMaker().softValues().evictionListener(this).makeMap(); + protected Cache buildFieldDataMap() { + CacheBuilder cacheBuilder = CacheBuilder.newBuilder().softValues().removalListener(this); + CacheBuilderHelper.disableStats(cacheBuilder); + return cacheBuilder.build(); } @Override @@ -60,7 +62,9 @@ public class SoftFieldDataCache extends AbstractConcurrentMapFieldDataCache impl } @Override - public void onEviction(@Nullable String s, @Nullable FieldData fieldData) { - evictions.inc(); + public void onRemoval(RemovalNotification removalNotification) { + if (removalNotification.wasEvicted()) { + evictions.inc(); + } } } diff --git a/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java b/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java index 7db2a88cf25..dc81261c5cf 100644 --- a/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java +++ b/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.cache.field.data.support; +import com.google.common.cache.Cache; import org.apache.lucene.index.IndexReader; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.common.settings.Settings; @@ -32,7 +33,6 @@ import org.elasticsearch.index.settings.IndexSettings; import java.io.IOException; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; /** @@ -40,15 +40,13 @@ import java.util.concurrent.ConcurrentMap; */ public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexComponent implements FieldDataCache, IndexReader.ReaderFinishedListener { - private final ConcurrentMap> cache; + private final ConcurrentMap> cache; private final Object creationMutex = new Object(); protected AbstractConcurrentMapFieldDataCache(Index index, @IndexSettings Settings indexSettings) { super(index, indexSettings); - // weak keys is fine, it will only be cleared once IndexReader references will be removed - // (assuming clear(...) will not be called) - this.cache = new ConcurrentHashMap>(); + this.cache = ConcurrentCollections.newConcurrentMap(); } @Override @@ -58,8 +56,8 @@ public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexC @Override public void clear(String fieldName) { - for (Map.Entry> entry : cache.entrySet()) { - entry.getValue().remove(fieldName); + for (Map.Entry> entry : cache.entrySet()) { + entry.getValue().invalidate(fieldName); } } @@ -75,19 +73,15 @@ public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexC @Override public void clear(IndexReader reader) { - ConcurrentMap map = cache.remove(reader.getCoreCacheKey()); - // help soft/weak handling GC - if (map != null) { - map.clear(); - } + cache.remove(reader.getCoreCacheKey()); } @Override public long sizeInBytes() { // the overhead of the map is not really relevant... long sizeInBytes = 0; - for (ConcurrentMap map : cache.values()) { - for (FieldData fieldData : map.values()) { + for (Cache map : cache.values()) { + for (FieldData fieldData : map.asMap().values()) { sizeInBytes += fieldData.sizeInBytes(); } } @@ -97,8 +91,8 @@ public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexC @Override public long sizeInBytes(String fieldName) { long sizeInBytes = 0; - for (ConcurrentMap map : cache.values()) { - FieldData fieldData = map.get(fieldName); + for (Cache map : cache.values()) { + FieldData fieldData = map.getIfPresent(fieldName); if (fieldData != null) { sizeInBytes += fieldData.sizeInBytes(); } @@ -108,7 +102,7 @@ public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexC @Override public FieldData cache(FieldDataType type, IndexReader reader, String fieldName) throws IOException { - ConcurrentMap fieldDataCache = cache.get(reader.getCoreCacheKey()); + Cache fieldDataCache = cache.get(reader.getCoreCacheKey()); if (fieldDataCache == null) { synchronized (creationMutex) { fieldDataCache = cache.get(reader.getCoreCacheKey()); @@ -119,10 +113,10 @@ public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexC } } } - FieldData fieldData = fieldDataCache.get(fieldName); + FieldData fieldData = fieldDataCache.getIfPresent(fieldName); if (fieldData == null) { synchronized (fieldDataCache) { - fieldData = fieldDataCache.get(fieldName); + fieldData = fieldDataCache.getIfPresent(fieldName); if (fieldData == null) { fieldData = FieldData.load(type, reader, fieldName); fieldDataCache.put(fieldName, fieldData); @@ -132,7 +126,5 @@ public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexC return fieldData; } - protected ConcurrentMap buildFieldDataMap() { - return ConcurrentCollections.newConcurrentMap(); - } + protected abstract Cache buildFieldDataMap(); } diff --git a/src/main/java/org/elasticsearch/index/cache/field/data/weak/WeakFieldDataCache.java b/src/main/java/org/elasticsearch/index/cache/field/data/weak/WeakFieldDataCache.java deleted file mode 100644 index a63cd313a85..00000000000 --- a/src/main/java/org/elasticsearch/index/cache/field/data/weak/WeakFieldDataCache.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.cache.field.data.weak; - -import com.google.common.collect.MapEvictionListener; -import com.google.common.collect.MapMaker; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.metrics.CounterMetric; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.cache.field.data.support.AbstractConcurrentMapFieldDataCache; -import org.elasticsearch.index.field.data.FieldData; -import org.elasticsearch.index.settings.IndexSettings; - -import java.util.concurrent.ConcurrentMap; - -/** - * - */ -public class WeakFieldDataCache extends AbstractConcurrentMapFieldDataCache implements MapEvictionListener { - - private final CounterMetric evictions = new CounterMetric(); - - @Inject - public WeakFieldDataCache(Index index, @IndexSettings Settings indexSettings) { - super(index, indexSettings); - } - - @Override - protected ConcurrentMap buildFieldDataMap() { - return new MapMaker().weakValues().evictionListener(this).makeMap(); - } - - @Override - public String type() { - return "weak"; - } - - @Override - public long evictions() { - return evictions.count(); - } - - @Override - public void onEviction(@Nullable String s, @Nullable FieldData fieldData) { - evictions.inc(); - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/index/cache/filter/FilterCacheModule.java b/src/main/java/org/elasticsearch/index/cache/filter/FilterCacheModule.java index c4144922e2d..c9c3d5a1a34 100644 --- a/src/main/java/org/elasticsearch/index/cache/filter/FilterCacheModule.java +++ b/src/main/java/org/elasticsearch/index/cache/filter/FilterCacheModule.java @@ -22,7 +22,7 @@ package org.elasticsearch.index.cache.filter; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.Scopes; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.cache.filter.node.NodeFilterCache; +import org.elasticsearch.index.cache.filter.weighted.WeightedFilterCache; /** * @@ -42,7 +42,7 @@ public class FilterCacheModule extends AbstractModule { @Override protected void configure() { bind(FilterCache.class) - .to(settings.getAsClass(FilterCacheSettings.FILTER_CACHE_TYPE, NodeFilterCache.class, "org.elasticsearch.index.cache.filter.", "FilterCache")) + .to(settings.getAsClass(FilterCacheSettings.FILTER_CACHE_TYPE, WeightedFilterCache.class, "org.elasticsearch.index.cache.filter.", "FilterCache")) .in(Scopes.SINGLETON); } } diff --git a/src/main/java/org/elasticsearch/index/cache/filter/node/NodeFilterCache.java b/src/main/java/org/elasticsearch/index/cache/filter/node/NodeFilterCache.java deleted file mode 100644 index 3af2a54ae63..00000000000 --- a/src/main/java/org/elasticsearch/index/cache/filter/node/NodeFilterCache.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.cache.filter.node; - -import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.lucene.docset.DocSet; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.cache.filter.support.AbstractWeightedFilterCache; -import org.elasticsearch.index.cache.filter.support.FilterCacheValue; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.indices.cache.filter.IndicesNodeFilterCache; - -import java.util.concurrent.ConcurrentMap; - -public class NodeFilterCache extends AbstractWeightedFilterCache { - - private final IndicesNodeFilterCache indicesNodeFilterCache; - - @Inject - public NodeFilterCache(Index index, @IndexSettings Settings indexSettings, IndicesNodeFilterCache indicesNodeFilterCache) { - super(index, indexSettings); - this.indicesNodeFilterCache = indicesNodeFilterCache; - - indicesNodeFilterCache.addEvictionListener(this); - } - - @Override - public void close() throws ElasticSearchException { - indicesNodeFilterCache.removeEvictionListener(this); - super.close(); - } - - @Override - protected ConcurrentMap> cache() { - return indicesNodeFilterCache.cache(); - } - - @Override - public String type() { - return "node"; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/index/cache/filter/resident/ResidentFilterCache.java b/src/main/java/org/elasticsearch/index/cache/filter/resident/ResidentFilterCache.java deleted file mode 100644 index c45930a16ad..00000000000 --- a/src/main/java/org/elasticsearch/index/cache/filter/resident/ResidentFilterCache.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.cache.filter.resident; - -import com.google.common.base.Objects; -import com.google.common.collect.MapEvictionListener; -import com.google.common.collect.MapMaker; -import org.apache.lucene.search.Filter; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.lucene.docset.DocSet; -import org.elasticsearch.common.metrics.CounterMetric; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.cache.filter.support.AbstractConcurrentMapFilterCache; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.settings.IndexSettingsService; - -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; - -/** - * A resident reference based filter cache that has weak keys on the IndexReader. - * - * - */ -public class ResidentFilterCache extends AbstractConcurrentMapFilterCache implements MapEvictionListener { - - private final IndexSettingsService indexSettingsService; - - private volatile int maxSize; - private volatile TimeValue expire; - - private final CounterMetric evictions = new CounterMetric(); - - private final ApplySettings applySettings = new ApplySettings(); - - @Inject - public ResidentFilterCache(Index index, @IndexSettings Settings indexSettings, IndexSettingsService indexSettingsService) { - super(index, indexSettings); - this.indexSettingsService = indexSettingsService; - this.maxSize = indexSettings.getAsInt("index.cache.filter.max_size", componentSettings.getAsInt("max_size", 1000)); - this.expire = indexSettings.getAsTime("index.cache.filter.expire", componentSettings.getAsTime("expire", null)); - logger.debug("using [resident] filter cache with max_size [{}], expire [{}]", maxSize, expire); - - indexSettingsService.addListener(applySettings); - } - - @Override - public void close() { - indexSettingsService.removeListener(applySettings); - super.close(); - } - - @Override - protected ConcurrentMap buildFilterMap() { - MapMaker mapMaker = new MapMaker(); - if (maxSize != -1) { - mapMaker.maximumSize(maxSize); - } - if (expire != null) { - mapMaker.expireAfterAccess(expire.nanos(), TimeUnit.NANOSECONDS); - } - mapMaker.evictionListener(this); - return mapMaker.makeMap(); - } - - @Override - public String type() { - return "resident"; - } - - @Override - public long evictions() { - return evictions.count(); - } - - @Override - public void onEviction(Filter filter, DocSet docSet) { - evictions.inc(); - } - - static { - IndexMetaData.addDynamicSettings( - "index.cache.field.max_size", - "index.cache.field.expire" - ); - } - - class ApplySettings implements IndexSettingsService.Listener { - @Override - public void onRefreshSettings(Settings settings) { - int maxSize = settings.getAsInt("index.cache.filter.max_size", ResidentFilterCache.this.maxSize); - TimeValue expire = settings.getAsTime("index.cache.filter.expire", ResidentFilterCache.this.expire); - boolean changed = false; - if (maxSize != ResidentFilterCache.this.maxSize) { - logger.info("updating index.cache.filter.max_size from [{}] to [{}]", ResidentFilterCache.this.maxSize, maxSize); - changed = true; - ResidentFilterCache.this.maxSize = maxSize; - } - if (!Objects.equal(expire, ResidentFilterCache.this.expire)) { - logger.info("updating index.cache.filter.expire from [{}] to [{}]", ResidentFilterCache.this.expire, expire); - changed = true; - ResidentFilterCache.this.expire = expire; - } - if (changed) { - clear(); - } - } - } -} diff --git a/src/main/java/org/elasticsearch/index/cache/filter/soft/SoftFilterCache.java b/src/main/java/org/elasticsearch/index/cache/filter/soft/SoftFilterCache.java deleted file mode 100644 index b132101ec2a..00000000000 --- a/src/main/java/org/elasticsearch/index/cache/filter/soft/SoftFilterCache.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.cache.filter.soft; - -import com.google.common.base.Objects; -import com.google.common.collect.MapEvictionListener; -import com.google.common.collect.MapMaker; -import org.apache.lucene.search.Filter; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.lucene.docset.DocSet; -import org.elasticsearch.common.metrics.CounterMetric; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.cache.filter.support.AbstractConcurrentMapFilterCache; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.settings.IndexSettingsService; - -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; - -/** - * A soft reference based filter cache that has soft keys on the IndexReader. - * - * - */ -public class SoftFilterCache extends AbstractConcurrentMapFilterCache implements MapEvictionListener { - - private final IndexSettingsService indexSettingsService; - - private volatile int maxSize; - private volatile TimeValue expire; - - private final CounterMetric evictions = new CounterMetric(); - - private final ApplySettings applySettings = new ApplySettings(); - - @Inject - public SoftFilterCache(Index index, @IndexSettings Settings indexSettings, IndexSettingsService indexSettingsService) { - super(index, indexSettings); - this.indexSettingsService = indexSettingsService; - this.maxSize = indexSettings.getAsInt("index.cache.filter.max_size", componentSettings.getAsInt("max_size", -1)); - this.expire = indexSettings.getAsTime("index.cache.filter.expire", componentSettings.getAsTime("expire", null)); - logger.debug("using [soft] filter cache with max_size [{}], expire [{}]", maxSize, expire); - - indexSettingsService.addListener(applySettings); - } - - @Override - public void close() { - indexSettingsService.removeListener(applySettings); - super.close(); - } - - @Override - protected ConcurrentMap buildFilterMap() { - // DocSet are not really stored with strong reference only when searching on them... - // Filter might be stored in query cache - MapMaker mapMaker = new MapMaker().softValues(); - if (maxSize != -1) { - mapMaker.maximumSize(maxSize); - } - if (expire != null && expire.nanos() > 0) { - mapMaker.expireAfterAccess(expire.nanos(), TimeUnit.NANOSECONDS); - } - mapMaker.evictionListener(this); - return mapMaker.makeMap(); - } - - @Override - public String type() { - return "soft"; - } - - @Override - public long evictions() { - return evictions.count(); - } - - @Override - public void onEviction(Filter filter, DocSet docSet) { - evictions.inc(); - } - - static { - IndexMetaData.addDynamicSettings( - "index.cache.field.max_size", - "index.cache.field.expire" - ); - } - - class ApplySettings implements IndexSettingsService.Listener { - @Override - public void onRefreshSettings(Settings settings) { - int maxSize = settings.getAsInt("index.cache.filter.max_size", SoftFilterCache.this.maxSize); - TimeValue expire = settings.getAsTime("index.cache.filter.expire", SoftFilterCache.this.expire); - boolean changed = false; - if (maxSize != SoftFilterCache.this.maxSize) { - logger.info("updating index.cache.filter.max_size from [{}] to [{}]", SoftFilterCache.this.maxSize, maxSize); - changed = true; - SoftFilterCache.this.maxSize = maxSize; - } - if (!Objects.equal(expire, SoftFilterCache.this.expire)) { - logger.info("updating index.cache.filter.expire from [{}] to [{}]", SoftFilterCache.this.expire, expire); - changed = true; - SoftFilterCache.this.expire = expire; - } - if (changed) { - clear(); - } - } - } -} diff --git a/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java b/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java deleted file mode 100644 index f68ab433331..00000000000 --- a/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.cache.filter.support; - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.DocIdSet; -import org.apache.lucene.search.Filter; -import org.elasticsearch.common.lucene.docset.DocSet; -import org.elasticsearch.common.lucene.search.NoCacheFilter; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.cache.filter.FilterCache; -import org.elasticsearch.index.settings.IndexSettings; - -import java.io.IOException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap; - -/** - * A base concurrent filter cache that accepts the actual cache to use. - */ -public abstract class AbstractConcurrentMapFilterCache extends AbstractIndexComponent implements FilterCache, IndexReader.ReaderFinishedListener { - - final ConcurrentMap>> cache; - - protected AbstractConcurrentMapFilterCache(Index index, @IndexSettings Settings indexSettings) { - super(index, indexSettings); - this.cache = buildCache(); - } - - protected ConcurrentMap>> buildCache() { - return new ConcurrentHashMap>>(); - } - - protected ConcurrentMap buildFilterMap() { - return newConcurrentMap(); - } - - @Override - public void close() { - cache.clear(); - } - - @Override - public void clear() { - cache.clear(); - } - - @Override - public void finished(IndexReader reader) { - FilterCacheValue> readerValue = cache.remove(reader.getCoreCacheKey()); - // help soft/weak handling GC - if (readerValue != null) { - readerValue.value().clear(); - } - } - - @Override - public void clear(IndexReader reader) { - FilterCacheValue> readerValue = cache.remove(reader.getCoreCacheKey()); - // help soft/weak handling GC - if (readerValue != null) { - readerValue.value().clear(); - } - } - - @Override - public EntriesStats entriesStats() { - long sizeInBytes = 0; - long totalCount = 0; - int segmentsCount = 0; - for (FilterCacheValue> readerValue : cache.values()) { - segmentsCount++; - for (DocSet docSet : readerValue.value().values()) { - sizeInBytes += docSet.sizeInBytes(); - totalCount++; - } - } - return new EntriesStats(sizeInBytes, segmentsCount == 0 ? 0 : totalCount / segmentsCount); - } - - @Override - public Filter cache(Filter filterToCache) { - if (filterToCache instanceof NoCacheFilter) { - return filterToCache; - } - if (isCached(filterToCache)) { - return filterToCache; - } - return new FilterCacheFilterWrapper(filterToCache, this); - } - - @Override - public boolean isCached(Filter filter) { - return filter instanceof FilterCacheFilterWrapper; - } - - // LUCENE MONITOR: Check next version Lucene for CachingWrapperFilter, consider using that logic - // and not use the DeletableConstantScoreQuery, instead pass the DeletesMode enum to the cache method - // see: https://issues.apache.org/jira/browse/LUCENE-2468 - - static class FilterCacheFilterWrapper extends Filter { - - private final Filter filter; - - private final AbstractConcurrentMapFilterCache cache; - - FilterCacheFilterWrapper(Filter filter, AbstractConcurrentMapFilterCache cache) { - this.filter = filter; - this.cache = cache; - } - - @Override - public DocIdSet getDocIdSet(IndexReader reader) throws IOException { - FilterCacheValue> cacheValue = cache.cache.get(reader.getCoreCacheKey()); - if (cacheValue == null) { - cacheValue = new FilterCacheValue>(cache.buildFilterMap()); - FilterCacheValue> prev = cache.cache.putIfAbsent(reader.getCoreCacheKey(), cacheValue); - if (prev != null) { - cacheValue = prev; - } else { - reader.addReaderFinishedListener(cache); - } - } - Object key = filter; - if (filter instanceof CacheKeyFilter) { - key = ((CacheKeyFilter) filter).cacheKey(); - } - - DocSet docSet = cacheValue.value().get(key); - if (docSet != null) { - return docSet; - } - DocIdSet docIdSet = filter.getDocIdSet(reader); - docSet = FilterCacheValue.cacheable(reader, docIdSet); - DocSet prev = cacheValue.value().putIfAbsent(key, docSet); - if (prev != null) { - docSet = prev; - } - return docSet == DocSet.EMPTY_DOC_SET ? null : docSet; - } - - public String toString() { - return "cache(" + filter + ")"; - } - - public boolean equals(Object o) { - if (!(o instanceof FilterCacheFilterWrapper)) return false; - return this.filter.equals(((FilterCacheFilterWrapper) o).filter); - } - - public int hashCode() { - return filter.hashCode() ^ 0x1117BF25; - } - } -} diff --git a/src/main/java/org/elasticsearch/index/cache/filter/weak/WeakFilterCache.java b/src/main/java/org/elasticsearch/index/cache/filter/weak/WeakFilterCache.java deleted file mode 100644 index a3b603e51af..00000000000 --- a/src/main/java/org/elasticsearch/index/cache/filter/weak/WeakFilterCache.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.cache.filter.weak; - -import com.google.common.base.Objects; -import com.google.common.collect.MapEvictionListener; -import com.google.common.collect.MapMaker; -import org.apache.lucene.search.Filter; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.lucene.docset.DocSet; -import org.elasticsearch.common.metrics.CounterMetric; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.cache.filter.support.AbstractConcurrentMapFilterCache; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.settings.IndexSettingsService; - -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; - -/** - * A weak reference based filter cache that has weak keys on the IndexReader. - * - * - */ -public class WeakFilterCache extends AbstractConcurrentMapFilterCache implements MapEvictionListener { - - private final IndexSettingsService indexSettingsService; - - private volatile int maxSize; - private volatile TimeValue expire; - - private final CounterMetric evictions = new CounterMetric(); - - private final ApplySettings applySettings = new ApplySettings(); - - @Inject - public WeakFilterCache(Index index, @IndexSettings Settings indexSettings, IndexSettingsService indexSettingsService) { - super(index, indexSettings); - this.indexSettingsService = indexSettingsService; - this.maxSize = indexSettings.getAsInt("index.cache.filter.max_size", componentSettings.getAsInt("max_size", -1)); - this.expire = indexSettings.getAsTime("index.cache.filter.expire", componentSettings.getAsTime("expire", null)); - logger.debug("using [weak] filter cache with max_size [{}], expire [{}]", maxSize, expire); - - indexSettingsService.addListener(applySettings); - } - - @Override - public void close() { - indexSettingsService.removeListener(applySettings); - super.close(); - } - - @Override - protected ConcurrentMap buildFilterMap() { - MapMaker mapMaker = new MapMaker().weakValues(); - if (maxSize != -1) { - mapMaker.maximumSize(maxSize); - } - if (expire != null) { - mapMaker.expireAfterAccess(expire.nanos(), TimeUnit.NANOSECONDS); - } - mapMaker.evictionListener(this); - return mapMaker.makeMap(); - } - - @Override - public String type() { - return "weak"; - } - - @Override - public long evictions() { - return evictions.count(); - } - - @Override - public void onEviction(Filter filter, DocSet docSet) { - evictions.inc(); - } - - static { - IndexMetaData.addDynamicSettings( - "index.cache.field.max_size", - "index.cache.field.expire" - ); - } - - class ApplySettings implements IndexSettingsService.Listener { - @Override - public void onRefreshSettings(Settings settings) { - int maxSize = settings.getAsInt("index.cache.filter.max_size", WeakFilterCache.this.maxSize); - TimeValue expire = settings.getAsTime("index.cache.filter.expire", WeakFilterCache.this.expire); - boolean changed = false; - if (maxSize != WeakFilterCache.this.maxSize) { - logger.info("updating index.cache.filter.max_size from [{}] to [{}]", WeakFilterCache.this.maxSize, maxSize); - changed = true; - WeakFilterCache.this.maxSize = maxSize; - } - if (!Objects.equal(expire, WeakFilterCache.this.expire)) { - logger.info("updating index.cache.filter.expire from [{}] to [{}]", WeakFilterCache.this.expire, expire); - changed = true; - WeakFilterCache.this.expire = expire; - } - if (changed) { - clear(); - } - } - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractWeightedFilterCache.java b/src/main/java/org/elasticsearch/index/cache/filter/weighted/WeightedFilterCache.java similarity index 67% rename from src/main/java/org/elasticsearch/index/cache/filter/support/AbstractWeightedFilterCache.java rename to src/main/java/org/elasticsearch/index/cache/filter/weighted/WeightedFilterCache.java index f0b50c3b7bc..20292e58829 100644 --- a/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractWeightedFilterCache.java +++ b/src/main/java/org/elasticsearch/index/cache/filter/weighted/WeightedFilterCache.java @@ -17,14 +17,17 @@ * under the License. */ -package org.elasticsearch.index.cache.filter.support; +package org.elasticsearch.index.cache.filter.weighted; -import com.googlecode.concurrentlinkedhashmap.EvictionListener; -import com.googlecode.concurrentlinkedhashmap.Weigher; +import com.google.common.cache.Cache; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; +import com.google.common.cache.Weigher; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.docset.DocSet; import org.elasticsearch.common.lucene.search.NoCacheFilter; import org.elasticsearch.common.metrics.CounterMetric; @@ -34,12 +37,17 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.Index; import org.elasticsearch.index.cache.filter.FilterCache; +import org.elasticsearch.index.cache.filter.support.CacheKeyFilter; +import org.elasticsearch.index.cache.filter.support.FilterCacheValue; import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.indices.cache.filter.IndicesFilterCache; import java.io.IOException; import java.util.concurrent.ConcurrentMap; -public abstract class AbstractWeightedFilterCache extends AbstractIndexComponent implements FilterCache, IndexReader.ReaderFinishedListener, EvictionListener> { +public class WeightedFilterCache extends AbstractIndexComponent implements FilterCache, IndexReader.ReaderFinishedListener, RemovalListener> { + + final IndicesFilterCache indicesFilterCache; final ConcurrentMap seenReaders = ConcurrentCollections.newConcurrentMap(); final CounterMetric seenReadersCount = new CounterMetric(); @@ -47,15 +55,22 @@ public abstract class AbstractWeightedFilterCache extends AbstractIndexComponent final CounterMetric evictionsMetric = new CounterMetric(); final MeanMetric totalMetric = new MeanMetric(); - protected AbstractWeightedFilterCache(Index index, @IndexSettings Settings indexSettings) { + @Inject + public WeightedFilterCache(Index index, @IndexSettings Settings indexSettings, IndicesFilterCache indicesFilterCache) { super(index, indexSettings); + this.indicesFilterCache = indicesFilterCache; + indicesFilterCache.addRemovalListener(index.name(), this); } - protected abstract ConcurrentMap> cache(); + @Override + public String type() { + return "weighted"; + } @Override public void close() throws ElasticSearchException { clear(); + indicesFilterCache.removeRemovalListener(index.name()); } @Override @@ -66,13 +81,10 @@ public abstract class AbstractWeightedFilterCache extends AbstractIndexComponent return; } seenReadersCount.dec(); - ConcurrentMap> cache = cache(); - for (FilterCacheKey key : cache.keySet()) { + for (FilterCacheKey key : indicesFilterCache.cache().asMap().keySet()) { if (key.readerKey() == readerKey) { - FilterCacheValue removed2 = cache.remove(key); - if (removed2 != null) { - totalMetric.dec(removed2.value().sizeInBytes()); - } + // invalidate will cause a removal and will be notified + indicesFilterCache.cache().invalidate(key); } } } @@ -92,13 +104,11 @@ public abstract class AbstractWeightedFilterCache extends AbstractIndexComponent return; } seenReadersCount.dec(); - ConcurrentMap> cache = cache(); - for (FilterCacheKey key : cache.keySet()) { + Cache> cache = indicesFilterCache.cache(); + for (FilterCacheKey key : cache.asMap().keySet()) { if (key.readerKey() == reader.getCoreCacheKey()) { - FilterCacheValue removed2 = cache.remove(key); - if (removed2 != null) { - totalMetric.dec(removed2.value().sizeInBytes()); - } + // invalidate will cause a removal and will be notified + cache.invalidate(key); } } } @@ -134,9 +144,9 @@ public abstract class AbstractWeightedFilterCache extends AbstractIndexComponent private final Filter filter; - private final AbstractWeightedFilterCache cache; + private final WeightedFilterCache cache; - FilterCacheFilterWrapper(Filter filter, AbstractWeightedFilterCache cache) { + FilterCacheFilterWrapper(Filter filter, WeightedFilterCache cache) { this.filter = filter; this.cache = cache; } @@ -147,10 +157,10 @@ public abstract class AbstractWeightedFilterCache extends AbstractIndexComponent if (filter instanceof CacheKeyFilter) { filterKey = ((CacheKeyFilter) filter).cacheKey(); } - FilterCacheKey cacheKey = new FilterCacheKey(reader.getCoreCacheKey(), filterKey); - ConcurrentMap> innerCache = cache.cache(); + FilterCacheKey cacheKey = new FilterCacheKey(cache.index().name(), reader.getCoreCacheKey(), filterKey); + Cache> innerCache = cache.indicesFilterCache.cache(); - FilterCacheValue cacheValue = innerCache.get(cacheKey); + FilterCacheValue cacheValue = innerCache.getIfPresent(cacheKey); if (cacheValue == null) { if (!cache.seenReaders.containsKey(reader.getCoreCacheKey())) { Boolean previous = cache.seenReaders.putIfAbsent(reader.getCoreCacheKey(), Boolean.TRUE); @@ -163,10 +173,10 @@ public abstract class AbstractWeightedFilterCache extends AbstractIndexComponent DocIdSet docIdSet = filter.getDocIdSet(reader); DocSet docSet = FilterCacheValue.cacheable(reader, docIdSet); cacheValue = new FilterCacheValue(docSet); - FilterCacheValue previous = innerCache.putIfAbsent(cacheKey, cacheValue); - if (previous == null) { - cache.totalMetric.inc(cacheValue.value().sizeInBytes()); - } + // we might put the same one concurrently, that's fine, it will be replaced and the removal + // will be called + cache.totalMetric.inc(cacheValue.value().sizeInBytes()); + innerCache.put(cacheKey, cacheValue); } return cacheValue.value() == DocSet.EMPTY_DOC_SET ? null : cacheValue.value(); @@ -187,39 +197,42 @@ public abstract class AbstractWeightedFilterCache extends AbstractIndexComponent } - // factored by 10 - public static class FilterCacheValueWeigher implements Weigher> { - - public static final long FACTOR = 10l; + public static class FilterCacheValueWeigher implements Weigher> { @Override - public int weightOf(FilterCacheValue value) { - int weight = (int) Math.min(value.value().sizeInBytes() / 10, Integer.MAX_VALUE); + public int weigh(FilterCacheKey key, FilterCacheValue value) { + int weight = (int) Math.min(value.value().sizeInBytes(), Integer.MAX_VALUE); return weight == 0 ? 1 : weight; } } + // this will only be called for our index / data, IndicesFilterCache makes sure it works like this based on the + // index we register the listener with @Override - public void onEviction(FilterCacheKey filterCacheKey, FilterCacheValue docSetFilterCacheValue) { - if (filterCacheKey != null) { - if (seenReaders.containsKey(filterCacheKey.readerKey())) { - evictionsMetric.inc(); - if (docSetFilterCacheValue != null) { - totalMetric.dec(docSetFilterCacheValue.value().sizeInBytes()); - } - } + public void onRemoval(RemovalNotification> removalNotification) { + if (removalNotification.wasEvicted()) { + evictionsMetric.inc(); + } + if (removalNotification.getValue() != null) { + totalMetric.dec(removalNotification.getValue().value().sizeInBytes()); } } public static class FilterCacheKey { + private final String index; private final Object readerKey; private final Object filterKey; - public FilterCacheKey(Object readerKey, Object filterKey) { + public FilterCacheKey(String index, Object readerKey, Object filterKey) { + this.index = index; this.readerKey = readerKey; this.filterKey = filterKey; } + public String index() { + return index; + } + public Object readerKey() { return readerKey; } diff --git a/src/main/java/org/elasticsearch/index/cache/query/parser/QueryParserCacheModule.java b/src/main/java/org/elasticsearch/index/cache/query/parser/QueryParserCacheModule.java index 4747d5e7455..5f60b81acbe 100644 --- a/src/main/java/org/elasticsearch/index/cache/query/parser/QueryParserCacheModule.java +++ b/src/main/java/org/elasticsearch/index/cache/query/parser/QueryParserCacheModule.java @@ -22,7 +22,7 @@ package org.elasticsearch.index.cache.query.parser; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.Scopes; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.cache.query.parser.weak.WeakQueryParserCache; +import org.elasticsearch.index.cache.query.parser.resident.ResidentQueryParserCache; /** * @@ -38,7 +38,7 @@ public class QueryParserCacheModule extends AbstractModule { @Override protected void configure() { bind(QueryParserCache.class) - .to(settings.getAsClass("index.cache.query.parser.type", WeakQueryParserCache.class, "org.elasticsearch.index.cache.query.parser.", "QueryParserCache")) + .to(settings.getAsClass("index.cache.query.parser.type", ResidentQueryParserCache.class, "org.elasticsearch.index.cache.query.parser.", "QueryParserCache")) .in(Scopes.SINGLETON); } } diff --git a/src/main/java/org/elasticsearch/index/cache/query/parser/resident/ResidentQueryParserCache.java b/src/main/java/org/elasticsearch/index/cache/query/parser/resident/ResidentQueryParserCache.java new file mode 100644 index 00000000000..b95bdbc7409 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/cache/query/parser/resident/ResidentQueryParserCache.java @@ -0,0 +1,86 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.cache.query.parser.resident; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.apache.lucene.queryParser.QueryParserSettings; +import org.apache.lucene.search.Query; +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.cache.CacheBuilderHelper; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.AbstractIndexComponent; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.cache.query.parser.QueryParserCache; +import org.elasticsearch.index.settings.IndexSettings; + +import java.util.concurrent.TimeUnit; + +/** + * A small (by default) query parser cache mainly to not parse the same query string several times + * if several shards exists on the same node. + */ +public class ResidentQueryParserCache extends AbstractIndexComponent implements QueryParserCache { + + private final Cache cache; + + private volatile int maxSize; + private volatile TimeValue expire; + + @Inject + public ResidentQueryParserCache(Index index, @IndexSettings Settings indexSettings) { + super(index, indexSettings); + + this.maxSize = indexSettings.getAsInt("index.cache.field.max_size", componentSettings.getAsInt("max_size", 100)); + this.expire = indexSettings.getAsTime("index.cache.field.expire", componentSettings.getAsTime("expire", null)); + logger.debug("using [resident] query cache with max_size [{}], expire [{}]", maxSize, expire); + + CacheBuilder cacheBuilder = CacheBuilder.newBuilder().maximumSize(maxSize); + if (expire != null) { + cacheBuilder.expireAfterAccess(expire.nanos(), TimeUnit.NANOSECONDS); + } + + CacheBuilderHelper.disableStats(cacheBuilder); + + this.cache = cacheBuilder.build(); + } + + @Override + public Query get(QueryParserSettings queryString) { + return cache.getIfPresent(queryString); + } + + @Override + public void put(QueryParserSettings queryString, Query query) { + cache.put(queryString, query); + } + + @Override + public void clear() { + cache.invalidateAll(); + } + + @Override + public void close() throws ElasticSearchException { + cache.invalidateAll(); + } +} diff --git a/src/main/java/org/elasticsearch/index/cache/query/parser/soft/SoftQueryParserCache.java b/src/main/java/org/elasticsearch/index/cache/query/parser/soft/SoftQueryParserCache.java deleted file mode 100644 index fa746fdff52..00000000000 --- a/src/main/java/org/elasticsearch/index/cache/query/parser/soft/SoftQueryParserCache.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.cache.query.parser.soft; - -import com.google.common.collect.MapMaker; -import org.apache.lucene.queryParser.QueryParserSettings; -import org.apache.lucene.search.Query; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.cache.query.parser.support.AbstractJvmQueryParserCache; -import org.elasticsearch.index.settings.IndexSettings; - -/** - * - */ -public class SoftQueryParserCache extends AbstractJvmQueryParserCache { - - @Inject - public SoftQueryParserCache(Index index, @IndexSettings Settings indexSettings) { - super(index, indexSettings, new MapMaker().softValues().makeMap()); - } -} diff --git a/src/main/java/org/elasticsearch/index/cache/query/parser/weak/WeakQueryParserCache.java b/src/main/java/org/elasticsearch/index/cache/query/parser/weak/WeakQueryParserCache.java deleted file mode 100644 index f819beada72..00000000000 --- a/src/main/java/org/elasticsearch/index/cache/query/parser/weak/WeakQueryParserCache.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.cache.query.parser.weak; - -import com.google.common.collect.MapMaker; -import org.apache.lucene.queryParser.QueryParserSettings; -import org.apache.lucene.search.Query; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.cache.query.parser.support.AbstractJvmQueryParserCache; -import org.elasticsearch.index.settings.IndexSettings; - -/** - * - */ -public class WeakQueryParserCache extends AbstractJvmQueryParserCache { - - @Inject - public WeakQueryParserCache(Index index, @IndexSettings Settings indexSettings) { - super(index, indexSettings, new MapMaker().weakValues().makeMap()); - } -} diff --git a/src/main/java/org/elasticsearch/indices/IndicesModule.java b/src/main/java/org/elasticsearch/indices/IndicesModule.java index 1aa5c91264f..032b8bec395 100644 --- a/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -25,7 +25,7 @@ import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.inject.SpawnModules; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.analysis.IndicesAnalysisModule; -import org.elasticsearch.indices.cache.filter.IndicesNodeFilterCache; +import org.elasticsearch.indices.cache.filter.IndicesFilterCache; import org.elasticsearch.indices.cluster.IndicesClusterStateService; import org.elasticsearch.indices.memory.IndexingMemoryController; import org.elasticsearch.indices.query.IndicesQueriesModule; @@ -63,7 +63,7 @@ public class IndicesModule extends AbstractModule implements SpawnModules { bind(IndicesClusterStateService.class).asEagerSingleton(); bind(IndexingMemoryController.class).asEagerSingleton(); - bind(IndicesNodeFilterCache.class).asEagerSingleton(); + bind(IndicesFilterCache.class).asEagerSingleton(); bind(TransportNodesListShardStoreMetaData.class).asEagerSingleton(); bind(IndicesTTLService.class).asEagerSingleton(); } diff --git a/src/main/java/org/elasticsearch/indices/cache/filter/IndicesFilterCache.java b/src/main/java/org/elasticsearch/indices/cache/filter/IndicesFilterCache.java new file mode 100644 index 00000000000..2fba6abc045 --- /dev/null +++ b/src/main/java/org/elasticsearch/indices/cache/filter/IndicesFilterCache.java @@ -0,0 +1,154 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices.cache.filter; + +import com.google.common.base.Objects; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; +import com.google.common.collect.ImmutableMap; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.cache.CacheBuilderHelper; +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.lucene.docset.DocSet; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.cache.filter.support.FilterCacheValue; +import org.elasticsearch.index.cache.filter.weighted.WeightedFilterCache; +import org.elasticsearch.monitor.jvm.JvmInfo; +import org.elasticsearch.node.settings.NodeSettingsService; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class IndicesFilterCache extends AbstractComponent implements RemovalListener> { + + private Cache> cache; + + private volatile String size; + private volatile long sizeInBytes; + private volatile TimeValue expire; + + private volatile Map>> removalListeners = + ImmutableMap.of(); + + + static { + MetaData.addDynamicSettings( + "indices.cache.filter.size", + "indices.cache.filter.expire" + ); + } + + class ApplySettings implements NodeSettingsService.Listener { + @Override + public void onRefreshSettings(Settings settings) { + boolean replace = false; + String size = settings.get("indices.cache.filter.size", IndicesFilterCache.this.size); + if (!size.equals(IndicesFilterCache.this.size)) { + logger.info("updating [indices.cache.filter.size] from [{}] to [{}]", IndicesFilterCache.this.size, size); + IndicesFilterCache.this.size = size; + replace = true; + } + TimeValue expire = settings.getAsTime("indices.cache.filter.expire", IndicesFilterCache.this.expire); + if (!Objects.equal(expire, IndicesFilterCache.this.expire)) { + logger.info("updating [indices.cache.filter.expire] from [{}] to [{}]", IndicesFilterCache.this.expire, expire); + IndicesFilterCache.this.expire = expire; + replace = true; + } + if (replace) { + Cache> oldCache = IndicesFilterCache.this.cache; + computeSizeInBytes(); + buildCache(); + oldCache.invalidateAll(); + } + } + } + + @Inject + public IndicesFilterCache(Settings settings, NodeSettingsService nodeSettingsService) { + super(settings); + this.size = componentSettings.get("size", "20%"); + this.expire = componentSettings.getAsTime("expire", null); + computeSizeInBytes(); + buildCache(); + logger.debug("using [node] filter cache with size [{}], actual_size [{}]", size, new ByteSizeValue(sizeInBytes)); + + nodeSettingsService.addListener(new ApplySettings()); + } + + private void buildCache() { + CacheBuilder> cacheBuilder = CacheBuilder.newBuilder() + .removalListener(this) + .maximumWeight(sizeInBytes).weigher(new WeightedFilterCache.FilterCacheValueWeigher()); + + // defaults to 4, but this is a busy map for all indices, increase it a bit + cacheBuilder.concurrencyLevel(8); + + if (expire != null) { + cacheBuilder.expireAfterAccess(expire.millis(), TimeUnit.MILLISECONDS); + } + + CacheBuilderHelper.disableStats(cacheBuilder); + + cache = cacheBuilder.build(); + } + + private void computeSizeInBytes() { + if (size.endsWith("%")) { + double percent = Double.parseDouble(size.substring(0, size.length() - 1)); + sizeInBytes = (long) ((percent / 100) * JvmInfo.jvmInfo().getMem().getHeapMax().bytes()); + } else { + sizeInBytes = ByteSizeValue.parseBytesSizeValue(size).bytes(); + } + } + + public synchronized void addRemovalListener(String index, RemovalListener> listener) { + removalListeners = MapBuilder.newMapBuilder(removalListeners).put(index, listener).immutableMap(); + } + + public synchronized void removeRemovalListener(String index) { + removalListeners = MapBuilder.newMapBuilder(removalListeners).remove(index).immutableMap(); + } + + public void close() { + cache.invalidateAll(); + } + + public Cache> cache() { + return this.cache; + } + + @Override + public void onRemoval(RemovalNotification> removalNotification) { + WeightedFilterCache.FilterCacheKey key = removalNotification.getKey(); + if (key == null) { + return; + } + RemovalListener> listener = removalListeners.get(key.index()); + if (listener != null) { + listener.onRemoval(removalNotification); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/indices/cache/filter/IndicesNodeFilterCache.java b/src/main/java/org/elasticsearch/indices/cache/filter/IndicesNodeFilterCache.java deleted file mode 100644 index 0f8e3252ac0..00000000000 --- a/src/main/java/org/elasticsearch/indices/cache/filter/IndicesNodeFilterCache.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.indices.cache.filter; - -import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; -import com.googlecode.concurrentlinkedhashmap.EvictionListener; -import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.lucene.docset.DocSet; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.cache.filter.support.AbstractWeightedFilterCache; -import org.elasticsearch.index.cache.filter.support.FilterCacheValue; -import org.elasticsearch.monitor.jvm.JvmInfo; -import org.elasticsearch.node.settings.NodeSettingsService; -import org.elasticsearch.threadpool.ThreadPool; - -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.TimeUnit; - -public class IndicesNodeFilterCache extends AbstractComponent implements EvictionListener> { - - private final ThreadPool threadPool; - - private ConcurrentMap> cache; - - private volatile String size; - private volatile long sizeInBytes; - - private final CopyOnWriteArrayList>> evictionListeners = - new CopyOnWriteArrayList>>(); - - @Inject - public IndicesNodeFilterCache(Settings settings, ThreadPool threadPool, NodeSettingsService nodeSettingsService) { - super(settings); - this.threadPool = threadPool; - this.size = componentSettings.get("size", "20%"); - computeSizeInBytes(); - buildCache(); - logger.debug("using [node] filter cache with size [{}], actual_size [{}]", size, new ByteSizeValue(sizeInBytes)); - - nodeSettingsService.addListener(new ApplySettings()); - } - - private void buildCache() { - TimeValue catchupTime = componentSettings.getAsTime("catchup", TimeValue.timeValueSeconds(10)); - - int weightedSize = (int) Math.min(sizeInBytes / AbstractWeightedFilterCache.FilterCacheValueWeigher.FACTOR, Integer.MAX_VALUE); - - cache = new ConcurrentLinkedHashMap.Builder>() - .maximumWeightedCapacity(weightedSize) - .weigher(new AbstractWeightedFilterCache.FilterCacheValueWeigher()) - .listener(this) - .catchup(this.threadPool.scheduler(), catchupTime.millis(), TimeUnit.MILLISECONDS) - .build(); - } - - private void computeSizeInBytes() { - if (size.endsWith("%")) { - double percent = Double.parseDouble(size.substring(0, size.length() - 1)); - sizeInBytes = (long) ((percent / 100) * JvmInfo.jvmInfo().getMem().getHeapMax().bytes()); - } else { - sizeInBytes = ByteSizeValue.parseBytesSizeValue(size).bytes(); - } - } - - public void addEvictionListener(EvictionListener> listener) { - evictionListeners.add(listener); - } - - public void removeEvictionListener(EvictionListener> listener) { - evictionListeners.remove(listener); - } - - public void close() { - cache.clear(); - } - - public ConcurrentMap> cache() { - return this.cache; - } - - @Override - public void onEviction(AbstractWeightedFilterCache.FilterCacheKey filterCacheKey, FilterCacheValue docSetFilterCacheValue) { - for (EvictionListener> listener : evictionListeners) { - listener.onEviction(filterCacheKey, docSetFilterCacheValue); - } - } - - class ApplySettings implements NodeSettingsService.Listener { - @Override - public void onRefreshSettings(Settings settings) { - String size = settings.get("indices.cache.filter.size", IndicesNodeFilterCache.this.size); - if (!size.equals(IndicesNodeFilterCache.this.size)) { - logger.info("updating [indices.cache.filter.size] from [{}] to [{}]", IndicesNodeFilterCache.this.size, size); - IndicesNodeFilterCache.this.size = size; - ConcurrentMap> oldCache = IndicesNodeFilterCache.this.cache; - computeSizeInBytes(); - buildCache(); - oldCache.clear(); - } - } - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/node/internal/InternalNode.java b/src/main/java/org/elasticsearch/node/internal/InternalNode.java index 6a9bc70aae5..99f33e61c6a 100644 --- a/src/main/java/org/elasticsearch/node/internal/InternalNode.java +++ b/src/main/java/org/elasticsearch/node/internal/InternalNode.java @@ -59,7 +59,7 @@ import org.elasticsearch.http.HttpServer; import org.elasticsearch.http.HttpServerModule; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.indices.cache.filter.IndicesNodeFilterCache; +import org.elasticsearch.indices.cache.filter.IndicesFilterCache; import org.elasticsearch.indices.cluster.IndicesClusterStateService; import org.elasticsearch.indices.memory.IndexingMemoryController; import org.elasticsearch.indices.ttl.IndicesTTLService; @@ -271,7 +271,7 @@ public final class InternalNode implements Node { stopWatch.stop().start("indices_cluster"); injector.getInstance(IndicesClusterStateService.class).close(); stopWatch.stop().start("indices"); - injector.getInstance(IndicesNodeFilterCache.class).close(); + injector.getInstance(IndicesFilterCache.class).close(); injector.getInstance(IndexingMemoryController.class).close(); injector.getInstance(IndicesTTLService.class).close(); injector.getInstance(IndicesService.class).close(); diff --git a/src/main/java/org/elasticsearch/script/ScriptService.java b/src/main/java/org/elasticsearch/script/ScriptService.java index c2615cf1f72..5542be73f3a 100644 --- a/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/src/main/java/org/elasticsearch/script/ScriptService.java @@ -19,9 +19,10 @@ package org.elasticsearch.script; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.MapMaker; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; @@ -54,7 +55,8 @@ public class ScriptService extends AbstractComponent { private final ConcurrentMap staticCache = ConcurrentCollections.newConcurrentMap(); - private final ConcurrentMap cache = new MapMaker().softValues().makeMap(); + // TODO expose some cache aspects like expiration and max size + private final Cache cache = CacheBuilder.newBuilder().build(); public ScriptService(Settings settings) { this(settings, new Environment(), ImmutableSet.builder() @@ -141,7 +143,7 @@ public class ScriptService extends AbstractComponent { lang = defaultLang; } CacheKey cacheKey = new CacheKey(lang, script); - compiled = cache.get(cacheKey); + compiled = cache.getIfPresent(cacheKey); if (compiled != null) { return compiled; } @@ -180,7 +182,7 @@ public class ScriptService extends AbstractComponent { } public void clear() { - cache.clear(); + cache.invalidateAll(); } public static class CacheKey { diff --git a/src/test/java/org/elasticsearch/test/unit/index/cache/filter/FilterCacheTests.java b/src/test/java/org/elasticsearch/test/unit/index/cache/filter/FilterCacheTests.java index ffdd05098ff..de15b588179 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/cache/filter/FilterCacheTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/cache/filter/FilterCacheTests.java @@ -31,9 +31,6 @@ import org.elasticsearch.common.lucene.search.TermFilter; import org.elasticsearch.index.Index; import org.elasticsearch.index.cache.filter.FilterCache; import org.elasticsearch.index.cache.filter.none.NoneFilterCache; -import org.elasticsearch.index.cache.filter.soft.SoftFilterCache; -import org.elasticsearch.index.cache.filter.weak.WeakFilterCache; -import org.elasticsearch.index.settings.IndexSettingsService; import org.testng.annotations.Test; import java.io.IOException; @@ -55,16 +52,6 @@ public class FilterCacheTests { verifyCache(new NoneFilterCache(new Index("test"), EMPTY_SETTINGS)); } - @Test - public void testSoftCache() throws Exception { - verifyCache(new SoftFilterCache(new Index("test"), EMPTY_SETTINGS, new IndexSettingsService(new Index("test"), EMPTY_SETTINGS))); - } - - @Test - public void testWeakCache() throws Exception { - verifyCache(new WeakFilterCache(new Index("test"), EMPTY_SETTINGS, new IndexSettingsService(new Index("test"), EMPTY_SETTINGS))); - } - private void verifyCache(FilterCache filterCache) throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER)); From 2b125bd6b4a79c80fa712c691da9a95a3c2051b9 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 5 Jan 2012 20:45:36 +0200 Subject: [PATCH 116/270] master not discovered should use 503 response code --- .../discovery/MasterNotDiscoveredException.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/elasticsearch/discovery/MasterNotDiscoveredException.java b/src/main/java/org/elasticsearch/discovery/MasterNotDiscoveredException.java index a40944cd3f2..218cecf83ee 100644 --- a/src/main/java/org/elasticsearch/discovery/MasterNotDiscoveredException.java +++ b/src/main/java/org/elasticsearch/discovery/MasterNotDiscoveredException.java @@ -20,6 +20,7 @@ package org.elasticsearch.discovery; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.rest.RestStatus; /** * @@ -29,4 +30,9 @@ public class MasterNotDiscoveredException extends ElasticSearchException { public MasterNotDiscoveredException() { super(""); } + + @Override + public RestStatus status() { + return RestStatus.SERVICE_UNAVAILABLE; + } } From d3b2a2cf66a816258c286400a4735194aed27155 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 5 Jan 2012 20:48:09 +0200 Subject: [PATCH 117/270] use a constant value for default master node timeout --- .../action/support/master/MasterNodeOperationRequest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequest.java b/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequest.java index 5be65a3ef9b..84509d264b7 100644 --- a/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequest.java +++ b/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequest.java @@ -28,12 +28,12 @@ import java.io.IOException; /** * A based request for master based operation. - * - * */ public abstract class MasterNodeOperationRequest implements ActionRequest { - private TimeValue masterNodeTimeout = TimeValue.timeValueSeconds(30); + public static TimeValue DEFAULT_MASTER_NODE_TIMEOUT = TimeValue.timeValueSeconds(30); + + private TimeValue masterNodeTimeout = DEFAULT_MASTER_NODE_TIMEOUT; @Override public boolean listenerThreaded() { From 759d94833554494887fa74fa9c1e9b41c19de7e3 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 5 Jan 2012 20:49:42 +0200 Subject: [PATCH 118/270] when setting timeout on cluster health, automatically set the master node timeout as well if it was not explicitly set --- .../action/admin/cluster/health/ClusterHealthRequest.java | 3 +++ .../action/support/master/MasterNodeOperationRequest.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java b/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java index 768ae1f9563..8f445bda60a 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java @@ -70,6 +70,9 @@ public class ClusterHealthRequest extends MasterNodeOperationRequest { public ClusterHealthRequest timeout(TimeValue timeout) { this.timeout = timeout; + if (masterNodeTimeout == DEFAULT_MASTER_NODE_TIMEOUT) { + masterNodeTimeout = timeout; + } return this; } diff --git a/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequest.java b/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequest.java index 84509d264b7..9ac53394c55 100644 --- a/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequest.java +++ b/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequest.java @@ -33,7 +33,7 @@ public abstract class MasterNodeOperationRequest implements ActionRequest { public static TimeValue DEFAULT_MASTER_NODE_TIMEOUT = TimeValue.timeValueSeconds(30); - private TimeValue masterNodeTimeout = DEFAULT_MASTER_NODE_TIMEOUT; + protected TimeValue masterNodeTimeout = DEFAULT_MASTER_NODE_TIMEOUT; @Override public boolean listenerThreaded() { From abc782abbbab8075bf7d329946359f08561a5c84 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 5 Jan 2012 20:50:16 +0200 Subject: [PATCH 119/270] add an option to control teh master node timeout in cluster health request --- .../action/admin/cluster/health/RestClusterHealthAction.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java index bdda24d13c8..fe5939c2e1b 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java @@ -53,6 +53,7 @@ public class RestClusterHealthAction extends BaseRestHandler { ClusterHealthRequest clusterHealthRequest = clusterHealthRequest(RestActions.splitIndices(request.param("index"))); int level = 0; try { + clusterHealthRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterHealthRequest.masterNodeTimeout())); clusterHealthRequest.timeout(request.paramAsTime("timeout", clusterHealthRequest.timeout())); String waitForStatus = request.param("wait_for_status"); if (waitForStatus != null) { From e38d80288b77abf09bf85a925e6bf3704dd28bb0 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 5 Jan 2012 20:50:54 +0200 Subject: [PATCH 120/270] allow to set the master node timeout also in rest cluster state api --- .../rest/action/admin/cluster/state/RestClusterStateAction.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java index 11a32b79e8d..872d6ed5386 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java @@ -72,6 +72,7 @@ public class RestClusterStateAction extends BaseRestHandler { @Override public void handleRequest(final RestRequest request, final RestChannel channel) { final ClusterStateRequest clusterStateRequest = Requests.clusterStateRequest(); + clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout())); clusterStateRequest.filterNodes(request.paramAsBoolean("filter_nodes", clusterStateRequest.filterNodes())); clusterStateRequest.filterRoutingTable(request.paramAsBoolean("filter_routing_table", clusterStateRequest.filterRoutingTable())); clusterStateRequest.filterMetaData(request.paramAsBoolean("filter_metadata", clusterStateRequest.filterMetaData())); From fcb96fdd1bdfb7cd0ee5bc78286ac22f20d20583 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 5 Jan 2012 20:58:23 +0200 Subject: [PATCH 121/270] cluster block failure should return 503 status code --- .../elasticsearch/cluster/block/ClusterBlockException.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java b/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java index 1ee48055254..0151360c5f8 100644 --- a/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java +++ b/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java @@ -21,6 +21,7 @@ package org.elasticsearch.cluster.block; import com.google.common.collect.ImmutableSet; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.rest.RestStatus; /** * @@ -54,4 +55,9 @@ public class ClusterBlockException extends ElasticSearchException { } return sb.toString(); } + + @Override + public RestStatus status() { + return RestStatus.SERVICE_UNAVAILABLE; + } } From 13ad6015c48a0033123269f991812e75b16c0635 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 5 Jan 2012 23:24:47 +0200 Subject: [PATCH 122/270] associate a rest status with a cluster block, and derive the status based on the blocks a failure has --- .../transport/NoNodeAvailableException.java | 8 ++++++-- .../cluster/block/ClusterBlock.java | 12 +++++++++++- .../cluster/block/ClusterBlockException.java | 12 ++++++++++-- .../cluster/block/ClusterBlocks.java | 16 ++++++++++++++-- .../cluster/metadata/IndexMetaData.java | 3 ++- .../elasticsearch/cluster/metadata/MetaData.java | 3 ++- .../metadata/MetaDataStateIndexService.java | 3 ++- .../org/elasticsearch/discovery/Discovery.java | 3 ++- .../elasticsearch/gateway/GatewayService.java | 3 ++- 9 files changed, 51 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/elasticsearch/client/transport/NoNodeAvailableException.java b/src/main/java/org/elasticsearch/client/transport/NoNodeAvailableException.java index 34c49d097a6..e0af1f7bb7e 100644 --- a/src/main/java/org/elasticsearch/client/transport/NoNodeAvailableException.java +++ b/src/main/java/org/elasticsearch/client/transport/NoNodeAvailableException.java @@ -20,15 +20,19 @@ package org.elasticsearch.client.transport; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.rest.RestStatus; /** * An exception indicating no node is available to perform the operation. - * - * */ public class NoNodeAvailableException extends ElasticSearchException { public NoNodeAvailableException() { super("No node available"); } + + @Override + public RestStatus status() { + return RestStatus.SERVICE_UNAVAILABLE; + } } diff --git a/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java b/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java index ef4eb4ecfb0..4de7a9588b5 100644 --- a/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java +++ b/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.RestStatus; import java.io.IOException; import java.io.Serializable; @@ -43,14 +44,17 @@ public class ClusterBlock implements Serializable, Streamable, ToXContent { private boolean disableStatePersistence = false; + private RestStatus status; + ClusterBlock() { } - public ClusterBlock(int id, String description, boolean retryable, boolean disableStatePersistence, ClusterBlockLevel... levels) { + public ClusterBlock(int id, String description, boolean retryable, boolean disableStatePersistence, RestStatus status, ClusterBlockLevel... levels) { this.id = id; this.description = description; this.retryable = retryable; this.disableStatePersistence = disableStatePersistence; + this.status = status; this.levels = levels; } @@ -62,6 +66,10 @@ public class ClusterBlock implements Serializable, Streamable, ToXContent { return this.description; } + public RestStatus status() { + return this.status; + } + public ClusterBlockLevel[] levels() { return this.levels; } @@ -123,6 +131,7 @@ public class ClusterBlock implements Serializable, Streamable, ToXContent { } retryable = in.readBoolean(); disableStatePersistence = in.readBoolean(); + status = RestStatus.readFrom(in); } @Override @@ -135,6 +144,7 @@ public class ClusterBlock implements Serializable, Streamable, ToXContent { } out.writeBoolean(retryable); out.writeBoolean(disableStatePersistence); + RestStatus.writeTo(out, status); } public String toString() { diff --git a/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java b/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java index 0151360c5f8..0ed5493c3c2 100644 --- a/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java +++ b/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java @@ -51,13 +51,21 @@ public class ClusterBlockException extends ElasticSearchException { private static String buildMessage(ImmutableSet blocks) { StringBuilder sb = new StringBuilder("blocked by: "); for (ClusterBlock block : blocks) { - sb.append("[").append(block.id()).append("/").append(block.description()).append("];"); + sb.append("[").append(block.status()).append("/").append(block.id()).append("/").append(block.description()).append("];"); } return sb.toString(); } @Override public RestStatus status() { - return RestStatus.SERVICE_UNAVAILABLE; + RestStatus status = null; + for (ClusterBlock block : blocks) { + if (status == null) { + status = block.status(); + } else if (status.getStatus() < block.status().getStatus()) { + status = block.status(); + } + } + return status; } } diff --git a/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java b/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java index 2189ff28d64..c71e9398675 100644 --- a/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java +++ b/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java @@ -32,8 +32,6 @@ import java.util.Set; /** * Represents current cluster level blocks to block dirty operations done against the cluster. - * - * */ public class ClusterBlocks { @@ -111,6 +109,20 @@ public class ClusterBlocks { return indicesBlocks.containsKey(index) && indicesBlocks.get(index).contains(block); } + public void globalBlockedRaiseException(ClusterBlockLevel level) throws ClusterBlockException { + ClusterBlockException blockException = globalBlockedException(level); + if (blockException != null) { + throw blockException; + } + } + + public ClusterBlockException globalBlockedException(ClusterBlockLevel level) { + if (global(level).isEmpty()) { + return null; + } + return new ClusterBlockException(ImmutableSet.copyOf(global(level))); + } + public void indexBlockedRaiseException(ClusterBlockLevel level, String index) throws ClusterBlockException { ClusterBlockException blockException = indexBlockedException(level, index); if (blockException != null) { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index cf01f742fd6..46a77fe43ec 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.RestStatus; import java.io.IOException; import java.util.Arrays; @@ -56,7 +57,7 @@ public class IndexMetaData { .add(IndexMetaData.SETTING_READ_ONLY) .build(); - public static final ClusterBlock INDEX_READ_ONLY_BLOCK = new ClusterBlock(5, "index read-only (api)", false, false, ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA); + public static final ClusterBlock INDEX_READ_ONLY_BLOCK = new ClusterBlock(5, "index read-only (api)", false, false, RestStatus.FORBIDDEN, ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA); public static ImmutableSet dynamicSettings() { return dynamicSettings; diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index cdc6728ec93..16ff991aacd 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.*; import org.elasticsearch.index.Index; import org.elasticsearch.indices.IndexMissingException; +import org.elasticsearch.rest.RestStatus; import java.io.IOException; import java.util.*; @@ -51,7 +52,7 @@ import static org.elasticsearch.common.settings.ImmutableSettings.*; public class MetaData implements Iterable { public static final String SETTING_READ_ONLY = "cluster.blocks.read_only"; - public static final ClusterBlock CLUSTER_READ_ONLY_BLOCK = new ClusterBlock(6, "cluster read-only (api)", false, false, ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA); + public static final ClusterBlock CLUSTER_READ_ONLY_BLOCK = new ClusterBlock(6, "cluster read-only (api)", false, false, RestStatus.FORBIDDEN, ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA); private static ImmutableSet dynamicSettings = ImmutableSet.builder() .add(SETTING_READ_ONLY) diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataStateIndexService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataStateIndexService.java index 5a30e233768..6a6d89996f0 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataStateIndexService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataStateIndexService.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.indices.IndexMissingException; +import org.elasticsearch.rest.RestStatus; import static org.elasticsearch.cluster.ClusterState.newClusterStateBuilder; @@ -43,7 +44,7 @@ import static org.elasticsearch.cluster.ClusterState.newClusterStateBuilder; */ public class MetaDataStateIndexService extends AbstractComponent { - public static final ClusterBlock INDEX_CLOSED_BLOCK = new ClusterBlock(4, "index closed", false, false, ClusterBlockLevel.READ_WRITE); + public static final ClusterBlock INDEX_CLOSED_BLOCK = new ClusterBlock(4, "index closed", false, false, RestStatus.FORBIDDEN, ClusterBlockLevel.READ_WRITE); private final ClusterService clusterService; diff --git a/src/main/java/org/elasticsearch/discovery/Discovery.java b/src/main/java/org/elasticsearch/discovery/Discovery.java index c702d98c4b1..373b7e79bea 100644 --- a/src/main/java/org/elasticsearch/discovery/Discovery.java +++ b/src/main/java/org/elasticsearch/discovery/Discovery.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.internal.Nullable; import org.elasticsearch.node.service.NodeService; +import org.elasticsearch.rest.RestStatus; /** * A pluggable module allowing to implement discovery of other nodes, publishing of the cluster @@ -34,7 +35,7 @@ import org.elasticsearch.node.service.NodeService; */ public interface Discovery extends LifecycleComponent { - final ClusterBlock NO_MASTER_BLOCK = new ClusterBlock(2, "no master", true, true, ClusterBlockLevel.ALL); + final ClusterBlock NO_MASTER_BLOCK = new ClusterBlock(2, "no master", true, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL); DiscoveryNode localNode(); diff --git a/src/main/java/org/elasticsearch/gateway/GatewayService.java b/src/main/java/org/elasticsearch/gateway/GatewayService.java index 47683331930..b415cc8d33e 100644 --- a/src/main/java/org/elasticsearch/gateway/GatewayService.java +++ b/src/main/java/org/elasticsearch/gateway/GatewayService.java @@ -39,6 +39,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.DiscoveryService; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; import java.util.Map; @@ -53,7 +54,7 @@ import static org.elasticsearch.cluster.metadata.MetaData.newMetaDataBuilder; */ public class GatewayService extends AbstractLifecycleComponent implements ClusterStateListener { - public static final ClusterBlock STATE_NOT_RECOVERED_BLOCK = new ClusterBlock(1, "state not recovered / initialized", true, true, ClusterBlockLevel.ALL); + public static final ClusterBlock STATE_NOT_RECOVERED_BLOCK = new ClusterBlock(1, "state not recovered / initialized", true, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL); private final Gateway gateway; From 93cce59a74305ad40901178b00fdc0217c371ae8 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 5 Jan 2012 23:45:00 +0200 Subject: [PATCH 123/270] have retry on conflict parameter default to 0, and really mean it when retrying... --- .../elasticsearch/action/update/TransportUpdateAction.java | 4 ++-- .../java/org/elasticsearch/action/update/UpdateRequest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java index 63d3732728f..4c5034bb5e5 100644 --- a/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java +++ b/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -204,7 +204,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio public void onFailure(Throwable e) { e = ExceptionsHelper.unwrapCause(e); if (e instanceof VersionConflictEngineException) { - if ((retryCount + 1) < request.retryOnConflict()) { + if (retryCount < request.retryOnConflict()) { threadPool.executor(executor()).execute(new Runnable() { @Override public void run() { @@ -231,7 +231,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio public void onFailure(Throwable e) { e = ExceptionsHelper.unwrapCause(e); if (e instanceof VersionConflictEngineException) { - if ((retryCount + 1) < request.retryOnConflict()) { + if (retryCount < request.retryOnConflict()) { threadPool.executor(executor()).execute(new Runnable() { @Override public void run() { diff --git a/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 3e1e7a62635..05e39cee924 100644 --- a/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -49,7 +49,7 @@ public class UpdateRequest extends InstanceShardOperationRequest { @Nullable Map scriptParams; - int retryOnConflict = 1; + int retryOnConflict = 0; private ReplicationType replicationType = ReplicationType.DEFAULT; private WriteConsistencyLevel consistencyLevel = WriteConsistencyLevel.DEFAULT; From ec8b7c3e233703b7eb4908fae74c3c38c0cd9d82 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 6 Jan 2012 23:38:41 +0200 Subject: [PATCH 124/270] No master (startup / minimum_master_node) / not recovered blocks should cause proper failures on operations, closes #1589. --- .../action/UnavailableShardsException.java | 9 +- .../TransportBroadcastPingAction.java | 14 +- .../TransportIndexReplicationPingAction.java | 13 ++ .../TransportReplicationPingAction.java | 13 ++ .../TransportShardReplicationPingAction.java | 12 ++ .../single/TransportSinglePingAction.java | 14 +- .../analyze/TransportAnalyzeAction.java | 21 ++- .../TransportClearIndicesCacheAction.java | 15 ++- .../indices/flush/TransportFlushAction.java | 14 +- .../TransportGatewaySnapshotAction.java | 15 ++- .../optimize/TransportOptimizeAction.java | 14 +- .../refresh/TransportRefreshAction.java | 14 +- .../TransportIndicesSegmentsAction.java | 14 +- .../stats/TransportIndicesStatsAction.java | 15 ++- .../status/TransportIndicesStatusAction.java | 16 ++- .../query/TransportValidateQueryAction.java | 14 +- .../action/bulk/TransportBulkAction.java | 5 + .../action/bulk/TransportShardBulkAction.java | 10 +- .../action/count/TransportCountAction.java | 14 +- .../action/delete/TransportDeleteAction.java | 30 +++-- .../index/TransportIndexDeleteAction.java | 10 +- .../index/TransportShardDeleteAction.java | 10 +- .../TransportDeleteByQueryAction.java | 12 +- .../TransportIndexDeleteByQueryAction.java | 10 +- .../TransportShardDeleteByQueryAction.java | 10 +- .../action/get/TransportGetAction.java | 22 ++-- .../action/get/TransportMultiGetAction.java | 5 + .../get/TransportShardMultiGetAction.java | 18 ++- .../action/index/TransportIndexAction.java | 19 ++- .../percolate/TransportPercolateAction.java | 14 +- .../TransportBroadcastOperationAction.java | 23 ++-- ...nsportIndexReplicationOperationAction.java | 16 ++- ...portIndicesReplicationOperationAction.java | 17 ++- ...nsportShardReplicationOperationAction.java | 72 +++++++--- .../TransportSingleCustomOperationAction.java | 15 ++- ...ransportInstanceSingleOperationAction.java | 61 +++++---- .../TransportShardSingleOperationAction.java | 23 +++- .../action/update/TransportUpdateAction.java | 10 +- .../cluster/MinimumMasterNodesTests.java | 2 + .../cluster/NoMasterNodeTests.java | 123 ++++++++++++++++++ 40 files changed, 627 insertions(+), 151 deletions(-) create mode 100644 src/test/java/org/elasticsearch/test/integration/cluster/NoMasterNodeTests.java diff --git a/src/main/java/org/elasticsearch/action/UnavailableShardsException.java b/src/main/java/org/elasticsearch/action/UnavailableShardsException.java index 0ff6d95c3d3..30d3811dd04 100644 --- a/src/main/java/org/elasticsearch/action/UnavailableShardsException.java +++ b/src/main/java/org/elasticsearch/action/UnavailableShardsException.java @@ -20,14 +20,16 @@ package org.elasticsearch.action; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.Nullable; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.rest.RestStatus; /** * */ public class UnavailableShardsException extends ElasticSearchException { - public UnavailableShardsException(ShardId shardId, String message) { + public UnavailableShardsException(@Nullable ShardId shardId, String message) { super(buildMessage(shardId, message)); } @@ -37,4 +39,9 @@ public class UnavailableShardsException extends ElasticSearchException { } return "[" + shardId.index().name() + "][" + shardId.id() + "] " + message; } + + @Override + public RestStatus status() { + return RestStatus.SERVICE_UNAVAILABLE; + } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/TransportBroadcastPingAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/TransportBroadcastPingAction.java index ec3eb79e38b..9f869cd295b 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/TransportBroadcastPingAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/TransportBroadcastPingAction.java @@ -27,6 +27,8 @@ import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedE import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.inject.Inject; @@ -65,10 +67,20 @@ public class TransportBroadcastPingAction extends TransportBroadcastOperationAct } @Override - protected GroupShardsIterator shards(BroadcastPingRequest request, String[] concreteIndices, ClusterState clusterState) { + protected GroupShardsIterator shards(ClusterState clusterState, BroadcastPingRequest request, String[] concreteIndices) { return clusterService.operationRouting().searchShards(clusterState, request.indices(), concreteIndices, request.queryHint(), null, null); } + @Override + protected ClusterBlockException checkGlobalBlock(ClusterState state, BroadcastPingRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.READ); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, BroadcastPingRequest request, String[] concreteIndices) { + return null; + } + @Override protected BroadcastPingResponse newResponse(BroadcastPingRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) { int successfulShards = 0; diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportIndexReplicationPingAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportIndexReplicationPingAction.java index 65d0c903f53..267bffe3271 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportIndexReplicationPingAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportIndexReplicationPingAction.java @@ -21,6 +21,9 @@ package org.elasticsearch.action.admin.cluster.ping.replication; import org.elasticsearch.action.support.replication.TransportIndexReplicationOperationAction; import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.common.inject.Inject; @@ -85,4 +88,14 @@ public class TransportIndexReplicationPingAction extends TransportIndexReplicati protected ShardReplicationPingRequest newShardRequestInstance(IndexReplicationPingRequest indexRequest, int shardId) { return new ShardReplicationPingRequest(indexRequest, shardId); } + + @Override + protected ClusterBlockException checkGlobalBlock(ClusterState state, IndexReplicationPingRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.READ); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, IndexReplicationPingRequest request) { + return state.blocks().indexBlockedException(ClusterBlockLevel.READ, request.index()); + } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportReplicationPingAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportReplicationPingAction.java index a29eb9c7621..53a0e15c031 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportReplicationPingAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportReplicationPingAction.java @@ -22,6 +22,9 @@ package org.elasticsearch.action.admin.cluster.ping.replication; import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.replication.TransportIndicesReplicationOperationAction; import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; @@ -70,4 +73,14 @@ public class TransportReplicationPingAction extends TransportIndicesReplicationO protected IndexReplicationPingRequest newIndexRequestInstance(ReplicationPingRequest request, String index, Set routing) { return new IndexReplicationPingRequest(request, index); } + + @Override + protected ClusterBlockException checkGlobalBlock(ClusterState state, ReplicationPingRequest replicationPingRequest) { + return state.blocks().globalBlockedException(ClusterBlockLevel.READ); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, ReplicationPingRequest replicationPingRequest, String[] concreteIndices) { + return state.blocks().indicesBlockedException(ClusterBlockLevel.WRITE, concreteIndices); + } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportShardReplicationPingAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportShardReplicationPingAction.java index b992491c60c..d6e53b0f432 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportShardReplicationPingAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportShardReplicationPingAction.java @@ -23,6 +23,8 @@ import org.elasticsearch.action.support.replication.TransportShardReplicationOpe import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.action.shard.ShardStateAction; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -81,6 +83,16 @@ public class TransportShardReplicationPingAction extends TransportShardReplicati protected void shardOperationOnReplica(ReplicaOperationRequest shardRequest) { } + @Override + protected ClusterBlockException checkGlobalBlock(ClusterState state, ShardReplicationPingRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.READ); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, ShardReplicationPingRequest request) { + return state.blocks().indexBlockedException(ClusterBlockLevel.READ, request.index()); + } + @Override protected ShardIterator shards(ClusterState clusterState, ShardReplicationPingRequest request) { return clusterService.state().routingTable().index(request.index()).shard(request.shardId()).shardsIt(); diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/TransportSinglePingAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/TransportSinglePingAction.java index 0c5191dbecd..69969593937 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/TransportSinglePingAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/TransportSinglePingAction.java @@ -24,6 +24,8 @@ import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.single.shard.TransportShardSingleOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -51,7 +53,17 @@ public class TransportSinglePingAction extends TransportShardSingleOperationActi } @Override - protected ShardIterator shards(ClusterState clusterState, SinglePingRequest request) throws ElasticSearchException { + protected ClusterBlockException checkGlobalBlock(ClusterState state, SinglePingRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.READ); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, SinglePingRequest singlePingRequest) { + return null; + } + + @Override + protected ShardIterator shards(ClusterState state, SinglePingRequest request) throws ElasticSearchException { return clusterService.operationRouting() .getShards(clusterService.state(), request.index(), request.type, request.id, null, null); } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index aa93fb76c07..1a7667419c5 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -32,6 +32,8 @@ import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.single.custom.TransportSingleCustomOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.routing.ShardsIterator; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.FastStringReader; @@ -87,13 +89,26 @@ public class TransportAnalyzeAction extends TransportSingleCustomOperationAction } @Override - protected ShardsIterator shards(ClusterState clusterState, AnalyzeRequest request) { + protected ClusterBlockException checkGlobalBlock(ClusterState state, AnalyzeRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.READ); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, AnalyzeRequest request) { + if (request.index() != null) { + request.index(state.metaData().concreteIndex(request.index())); + return state.blocks().indexBlockedException(ClusterBlockLevel.READ, request.index()); + } + return null; + } + + @Override + protected ShardsIterator shards(ClusterState state, AnalyzeRequest request) { if (request.index() == null) { // just execute locally.... return null; } - request.index(clusterState.metaData().concreteIndex(request.index())); - return clusterState.routingTable().index(request.index()).randomAllActiveShardsIt(); + return state.routingTable().index(request.index()).randomAllActiveShardsIt(); } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java b/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java index dcf4219eb48..767b594a199 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java @@ -27,6 +27,8 @@ import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedE import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.inject.Inject; @@ -160,7 +162,18 @@ public class TransportClearIndicesCacheAction extends TransportBroadcastOperatio * The refresh request works against *all* shards. */ @Override - protected GroupShardsIterator shards(ClearIndicesCacheRequest request, String[] concreteIndices, ClusterState clusterState) { + protected GroupShardsIterator shards(ClusterState clusterState, ClearIndicesCacheRequest request, String[] concreteIndices) { return clusterState.routingTable().allActiveShardsGrouped(concreteIndices, true); } + + @Override + protected ClusterBlockException checkGlobalBlock(ClusterState state, ClearIndicesCacheRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, ClearIndicesCacheRequest request, String[] concreteIndices) { + return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA, concreteIndices); + } + } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java b/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java index 45269e8a3e4..05965014b3f 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java @@ -27,6 +27,8 @@ import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedE import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.inject.Inject; @@ -123,7 +125,17 @@ public class TransportFlushAction extends TransportBroadcastOperationAction> routingMap = clusterState.metaData().resolveSearchRouting(Integer.toString(ThreadLocalRandom.current().nextInt(1000)), request.indices()); return clusterService.operationRouting().searchShards(clusterState, request.indices(), concreteIndices, null, routingMap, "_local"); } @Override - protected void checkBlock(ValidateQueryRequest request, String[] concreteIndices, ClusterState state) { - for (String index : concreteIndices) { - state.blocks().indexBlocked(ClusterBlockLevel.READ, index); - } + protected ClusterBlockException checkGlobalBlock(ClusterState state, ValidateQueryRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.READ); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, ValidateQueryRequest countRequest, String[] concreteIndices) { + return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, concreteIndices); } @Override diff --git a/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 84e37db2fcc..35dc277200d 100644 --- a/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -34,6 +34,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.BaseAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.GroupShardsIterator; @@ -138,6 +139,9 @@ public class TransportBulkAction extends BaseAction { private void executeBulk(final BulkRequest bulkRequest, final long startTime, final ActionListener listener) { ClusterState clusterState = clusterService.state(); + // TODO use timeout to wait here if its blocked... + clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.WRITE); + MetaData metaData = clusterState.metaData(); for (ActionRequest request : bulkRequest.requests) { if (request instanceof IndexRequest) { @@ -152,6 +156,7 @@ public class TransportBulkAction extends BaseAction { indexRequest.process(metaData, aliasOrIndex, mappingMd, allowIdGeneration); } else if (request instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) request; + deleteRequest.routing(clusterState.metaData().resolveIndexRouting(deleteRequest.routing(), deleteRequest.index())); deleteRequest.index(clusterState.metaData().concreteIndex(deleteRequest.index())); } } diff --git a/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index d397be17531..006615f4c0d 100644 --- a/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.action.index.MappingUpdatedAction; import org.elasticsearch.cluster.action.shard.ShardStateAction; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.routing.ShardIterator; @@ -107,8 +108,13 @@ public class TransportShardBulkAction extends TransportShardReplicationOperation } @Override - protected void checkBlock(BulkShardRequest request, ClusterState state) { - state.blocks().indexBlockedRaiseException(ClusterBlockLevel.WRITE, request.index()); + protected ClusterBlockException checkGlobalBlock(ClusterState state, BulkShardRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.WRITE); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, BulkShardRequest request) { + return state.blocks().indexBlockedException(ClusterBlockLevel.WRITE, request.index()); } @Override diff --git a/src/main/java/org/elasticsearch/action/count/TransportCountAction.java b/src/main/java/org/elasticsearch/action/count/TransportCountAction.java index a88fd542978..503351d4597 100644 --- a/src/main/java/org/elasticsearch/action/count/TransportCountAction.java +++ b/src/main/java/org/elasticsearch/action/count/TransportCountAction.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedE import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardRouting; @@ -89,16 +90,19 @@ public class TransportCountAction extends TransportBroadcastOperationAction> routingMap = clusterState.metaData().resolveSearchRouting(request.routing(), request.indices()); return clusterService.operationRouting().searchShards(clusterState, request.indices(), concreteIndices, request.queryHint(), routingMap, null); } @Override - protected void checkBlock(CountRequest request, String[] concreteIndices, ClusterState state) { - for (String index : concreteIndices) { - state.blocks().indexBlocked(ClusterBlockLevel.READ, index); - } + protected ClusterBlockException checkGlobalBlock(ClusterState state, CountRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.READ); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, CountRequest countRequest, String[] concreteIndices) { + return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, concreteIndices); } @Override diff --git a/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java b/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java index a67e90b2388..4870e164297 100644 --- a/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java +++ b/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java @@ -33,6 +33,7 @@ import org.elasticsearch.action.support.replication.TransportShardReplicationOpe import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.action.shard.ShardStateAction; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.routing.ShardIterator; @@ -47,8 +48,6 @@ import org.elasticsearch.transport.TransportService; /** * Performs the delete operation. - * - * */ public class TransportDeleteAction extends TransportShardReplicationOperationAction { @@ -98,13 +97,13 @@ public class TransportDeleteAction extends TransportShardReplicationOperationAct } } - private void innerExecute(final DeleteRequest request, final ActionListener listener) { - ClusterState clusterState = clusterService.state(); - request.routing(clusterState.metaData().resolveIndexRouting(request.routing(), request.index())); - request.index(clusterState.metaData().concreteIndex(request.index())); // we need to get the concrete index here... - if (clusterState.metaData().hasIndex(request.index())) { + @Override + protected boolean resolveRequest(final ClusterState state, final DeleteRequest request, final ActionListener listener) { + request.routing(state.metaData().resolveIndexRouting(request.routing(), request.index())); + request.index(state.metaData().concreteIndex(request.index())); + if (state.metaData().hasIndex(request.index())) { // check if routing is required, if so, do a broadcast delete - MappingMetaData mappingMd = clusterState.metaData().index(request.index()).mapping(request.type()); + MappingMetaData mappingMd = state.metaData().index(request.index()).mapping(request.type()); if (mappingMd != null && mappingMd.routing().required()) { if (request.routing() == null) { indexDeleteAction.execute(new IndexDeleteRequest(request), new ActionListener() { @@ -128,10 +127,14 @@ public class TransportDeleteAction extends TransportShardReplicationOperationAct listener.onFailure(e); } }); - return; + return false; } } } + return true; + } + + private void innerExecute(final DeleteRequest request, final ActionListener listener) { super.doExecute(request, listener); } @@ -161,8 +164,13 @@ public class TransportDeleteAction extends TransportShardReplicationOperationAct } @Override - protected void checkBlock(DeleteRequest request, ClusterState state) { - state.blocks().indexBlockedRaiseException(ClusterBlockLevel.WRITE, request.index()); + protected ClusterBlockException checkGlobalBlock(ClusterState state, DeleteRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.WRITE); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, DeleteRequest request) { + return state.blocks().indexBlockedException(ClusterBlockLevel.WRITE, request.index()); } @Override diff --git a/src/main/java/org/elasticsearch/action/delete/index/TransportIndexDeleteAction.java b/src/main/java/org/elasticsearch/action/delete/index/TransportIndexDeleteAction.java index c0fb6dc4e1f..fc8d099fadd 100644 --- a/src/main/java/org/elasticsearch/action/delete/index/TransportIndexDeleteAction.java +++ b/src/main/java/org/elasticsearch/action/delete/index/TransportIndexDeleteAction.java @@ -22,6 +22,7 @@ package org.elasticsearch.action.delete.index; import org.elasticsearch.action.support.replication.TransportIndexReplicationOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.common.inject.Inject; @@ -75,8 +76,13 @@ public class TransportIndexDeleteAction extends TransportIndexReplicationOperati } @Override - protected void checkBlock(IndexDeleteRequest request, ClusterState state) { - state.blocks().indexBlockedRaiseException(ClusterBlockLevel.WRITE, request.index()); + protected ClusterBlockException checkGlobalBlock(ClusterState state, IndexDeleteRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.WRITE); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, IndexDeleteRequest request) { + return state.blocks().indexBlockedException(ClusterBlockLevel.WRITE, request.index()); } @Override diff --git a/src/main/java/org/elasticsearch/action/delete/index/TransportShardDeleteAction.java b/src/main/java/org/elasticsearch/action/delete/index/TransportShardDeleteAction.java index 9a0c016e8b1..a595da63c8a 100644 --- a/src/main/java/org/elasticsearch/action/delete/index/TransportShardDeleteAction.java +++ b/src/main/java/org/elasticsearch/action/delete/index/TransportShardDeleteAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.support.replication.TransportShardReplicationOpe import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.action.shard.ShardStateAction; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; @@ -78,8 +79,13 @@ public class TransportShardDeleteAction extends TransportShardReplicationOperati } @Override - protected void checkBlock(ShardDeleteRequest request, ClusterState state) { - state.blocks().indexBlockedRaiseException(ClusterBlockLevel.WRITE, request.index()); + protected ClusterBlockException checkGlobalBlock(ClusterState state, ShardDeleteRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.WRITE); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, ShardDeleteRequest request) { + return state.blocks().indexBlockedException(ClusterBlockLevel.WRITE, request.index()); } @Override diff --git a/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java b/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java index e8d523dfb3d..587b9548c25 100644 --- a/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java +++ b/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.replication.TransportIndicesReplicationOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -70,10 +71,13 @@ public class TransportDeleteByQueryAction extends TransportIndicesReplicationOpe } @Override - protected void checkBlock(DeleteByQueryRequest request, String[] concreteIndices, ClusterState state) { - for (String index : concreteIndices) { - state.blocks().indexBlockedRaiseException(ClusterBlockLevel.WRITE, index); - } + protected ClusterBlockException checkGlobalBlock(ClusterState state, DeleteByQueryRequest replicationPingRequest) { + return state.blocks().globalBlockedException(ClusterBlockLevel.READ); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, DeleteByQueryRequest replicationPingRequest, String[] concreteIndices) { + return state.blocks().indicesBlockedException(ClusterBlockLevel.WRITE, concreteIndices); } @Override diff --git a/src/main/java/org/elasticsearch/action/deletebyquery/TransportIndexDeleteByQueryAction.java b/src/main/java/org/elasticsearch/action/deletebyquery/TransportIndexDeleteByQueryAction.java index 6c52fe0b64a..b815b38ee6d 100644 --- a/src/main/java/org/elasticsearch/action/deletebyquery/TransportIndexDeleteByQueryAction.java +++ b/src/main/java/org/elasticsearch/action/deletebyquery/TransportIndexDeleteByQueryAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.replication.TransportIndexReplicationOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.common.inject.Inject; @@ -73,8 +74,13 @@ public class TransportIndexDeleteByQueryAction extends TransportIndexReplication } @Override - protected void checkBlock(IndexDeleteByQueryRequest request, ClusterState state) { - state.blocks().indexBlockedRaiseException(ClusterBlockLevel.WRITE, request.index()); + protected ClusterBlockException checkGlobalBlock(ClusterState state, IndexDeleteByQueryRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.WRITE); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, IndexDeleteByQueryRequest request) { + return state.blocks().indexBlockedException(ClusterBlockLevel.WRITE, request.index()); } @Override diff --git a/src/main/java/org/elasticsearch/action/deletebyquery/TransportShardDeleteByQueryAction.java b/src/main/java/org/elasticsearch/action/deletebyquery/TransportShardDeleteByQueryAction.java index 8738aa17b48..40ae24e4c01 100644 --- a/src/main/java/org/elasticsearch/action/deletebyquery/TransportShardDeleteByQueryAction.java +++ b/src/main/java/org/elasticsearch/action/deletebyquery/TransportShardDeleteByQueryAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.support.replication.TransportShardReplicationOpe import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.action.shard.ShardStateAction; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; @@ -79,8 +80,13 @@ public class TransportShardDeleteByQueryAction extends TransportShardReplication } @Override - protected void checkBlock(ShardDeleteByQueryRequest request, ClusterState state) { - state.blocks().indexBlockedRaiseException(ClusterBlockLevel.WRITE, request.index()); + protected ClusterBlockException checkGlobalBlock(ClusterState state, ShardDeleteByQueryRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.WRITE); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, ShardDeleteByQueryRequest request) { + return state.blocks().indexBlockedException(ClusterBlockLevel.WRITE, request.index()); } @Override diff --git a/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/src/main/java/org/elasticsearch/action/get/TransportGetAction.java index e1d1eba6adc..87e4ed0710c 100644 --- a/src/main/java/org/elasticsearch/action/get/TransportGetAction.java +++ b/src/main/java/org/elasticsearch/action/get/TransportGetAction.java @@ -20,13 +20,12 @@ package org.elasticsearch.action.get; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.single.shard.TransportShardSingleOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -67,26 +66,29 @@ public class TransportGetAction extends TransportShardSingleOperationAction listener) { + protected void resolveRequest(ClusterState state, GetRequest request) { if (request.realtime == null) { request.realtime = this.realtime; } // update the routing (request#index here is possibly an alias) - MetaData metaData = clusterService.state().metaData(); - request.routing(metaData.resolveIndexRouting(request.routing(), request.index())); - - super.doExecute(request, listener); + request.routing(state.metaData().resolveIndexRouting(request.routing(), request.index())); + request.index(state.metaData().concreteIndex(request.index())); } @Override diff --git a/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java b/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java index 775a9632af3..3819bdb69a5 100644 --- a/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java +++ b/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.BaseAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.shard.ShardId; @@ -55,9 +56,13 @@ public class TransportMultiGetAction extends BaseAction listener) { ClusterState clusterState = clusterService.state(); + + clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ); + Map shardRequests = new HashMap(); for (int i = 0; i < request.items.size(); i++) { MultiGetRequest.Item item = request.items.get(i); + item.routing(clusterState.metaData().resolveIndexRouting(item.routing(), item.index())); item.index(clusterState.metaData().concreteIndex(item.index())); ShardId shardId = clusterService.operationRouting() .getShards(clusterState, item.index(), item.type(), item.id(), item.routing(), null).shardId(); diff --git a/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java index 8296d18e2ed..f3561fa7db2 100644 --- a/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java +++ b/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java @@ -21,11 +21,11 @@ package org.elasticsearch.action.get; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.single.shard.TransportShardSingleOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.common.inject.Inject; @@ -74,22 +74,28 @@ public class TransportShardMultiGetAction extends TransportShardSingleOperationA } @Override - protected void checkBlock(MultiGetShardRequest request, ClusterState state) { - state.blocks().indexBlockedRaiseException(ClusterBlockLevel.READ, request.index()); + protected ClusterBlockException checkGlobalBlock(ClusterState state, MultiGetShardRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.READ); } @Override - protected ShardIterator shards(ClusterState clusterState, MultiGetShardRequest request) { + protected ClusterBlockException checkRequestBlock(ClusterState state, MultiGetShardRequest request) { + return state.blocks().indexBlockedException(ClusterBlockLevel.READ, request.index()); + } + + @Override + protected ShardIterator shards(ClusterState state, MultiGetShardRequest request) { return clusterService.operationRouting() .getShards(clusterService.state(), request.index(), request.shardId(), request.preference()); } @Override - protected void doExecute(MultiGetShardRequest request, ActionListener listener) { + protected void resolveRequest(ClusterState state, MultiGetShardRequest request) { if (request.realtime == null) { request.realtime = this.realtime; } - super.doExecute(request, listener); + // no need to set concrete index and routing here, it has already been set by the multi get action on the item + //request.index(state.metaData().concreteIndex(request.index())); } @Override diff --git a/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java b/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java index f1ed9f5d846..7eab00a4fb3 100644 --- a/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java +++ b/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java @@ -31,6 +31,7 @@ import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.action.index.MappingUpdatedAction; import org.elasticsearch.cluster.action.shard.ShardStateAction; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; @@ -63,8 +64,6 @@ import java.util.concurrent.TimeUnit; * Defaults to true. *
  • allowIdGeneration: If the id is set not, should it be generated. Defaults to true. * - * - * */ public class TransportIndexAction extends TransportShardReplicationOperationAction { @@ -92,6 +91,7 @@ public class TransportIndexAction extends TransportShardReplicationOperationActi @Override protected void doExecute(final IndexRequest request, final ActionListener listener) { + // if we don't have a master, we don't have metadata, that's fine, let it find a master using create index API if (autoCreateIndex && !clusterService.state().metaData().hasConcreteIndex(request.index())) { request.beforeLocalFork(); // we fork on another thread... createIndexAction.execute(new CreateIndexRequest(request.index()).cause("auto(index api)"), new ActionListener() { @@ -119,7 +119,8 @@ public class TransportIndexAction extends TransportShardReplicationOperationActi } } - private void innerExecute(final IndexRequest request, final ActionListener listener) { + @Override + protected boolean resolveRequest(ClusterState state, IndexRequest request, ActionListener indexResponseActionListener) { MetaData metaData = clusterService.state().metaData(); String aliasOrIndex = request.index(); request.index(metaData.concreteIndex(request.index())); @@ -128,7 +129,10 @@ public class TransportIndexAction extends TransportShardReplicationOperationActi mappingMd = metaData.index(request.index()).mapping(request.type()); } request.process(metaData, aliasOrIndex, mappingMd, allowIdGeneration); + return true; + } + private void innerExecute(final IndexRequest request, final ActionListener listener) { super.doExecute(request, listener); } @@ -163,8 +167,13 @@ public class TransportIndexAction extends TransportShardReplicationOperationActi } @Override - protected void checkBlock(IndexRequest request, ClusterState state) { - state.blocks().indexBlockedRaiseException(ClusterBlockLevel.WRITE, request.index()); + protected ClusterBlockException checkGlobalBlock(ClusterState state, IndexRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.WRITE); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, IndexRequest request) { + return state.blocks().indexBlockedException(ClusterBlockLevel.WRITE, request.index()); } @Override diff --git a/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java b/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java index 5efe3f0fca7..560179eb7b8 100644 --- a/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java +++ b/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java @@ -24,6 +24,8 @@ import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.single.custom.TransportSingleCustomOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.routing.ShardsIterator; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -68,9 +70,19 @@ public class TransportPercolateAction extends TransportSingleCustomOperationActi return TransportActions.PERCOLATE; } + @Override + protected ClusterBlockException checkGlobalBlock(ClusterState state, PercolateRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.READ); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, PercolateRequest request) { + request.index(state.metaData().concreteIndex(request.index())); + return state.blocks().indexBlockedException(ClusterBlockLevel.READ, request.index()); + } + @Override protected ShardsIterator shards(ClusterState clusterState, PercolateRequest request) { - request.index(clusterState.metaData().concreteIndex(request.index())); return clusterState.routingTable().index(request.index()).randomAllActiveShardsIt(); } diff --git a/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastOperationAction.java b/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastOperationAction.java index d67f33b22fa..32e45a13816 100644 --- a/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastOperationAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.BaseAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.GroupShardsIterator; @@ -88,7 +89,7 @@ public abstract class TransportBroadcastOperationAction listener) { this.request = request; this.listener = listener; clusterState = clusterService.state(); + ClusterBlockException blockException = checkGlobalBlock(clusterState, request); + if (blockException != null) { + throw blockException; + } // update to concrete indices - concreteIndices = clusterState.metaData().concreteIndices(request.indices(), false, true); - checkBlock(request, concreteIndices, clusterState); + String[] concreteIndices = clusterState.metaData().concreteIndices(request.indices(), false, true); + blockException = checkRequestBlock(clusterState, request, concreteIndices); + if (blockException != null) { + throw blockException; + } nodes = clusterState.nodes(); - shardsIts = shards(request, concreteIndices, clusterState); + shardsIts = shards(clusterState, request, concreteIndices); expectedOps = shardsIts.size(); - shardsResponses = new AtomicReferenceArray(expectedOps); } diff --git a/src/main/java/org/elasticsearch/action/support/replication/TransportIndexReplicationOperationAction.java b/src/main/java/org/elasticsearch/action/support/replication/TransportIndexReplicationOperationAction.java index 41ff5c31cc9..43a1536d398 100644 --- a/src/main/java/org/elasticsearch/action/support/replication/TransportIndexReplicationOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/replication/TransportIndexReplicationOperationAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.support.BaseAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.common.inject.Inject; @@ -60,12 +61,17 @@ public abstract class TransportIndexReplicationOperationAction listener) { - ClusterState clusterState = clusterService.state(); + ClusterBlockException blockException = checkGlobalBlock(clusterState, request); + if (blockException != null) { + throw blockException; + } // update to concrete index request.index(clusterState.metaData().concreteIndex(request.index())); - - checkBlock(request, clusterState); + blockException = checkRequestBlock(clusterState, request); + if (blockException != null) { + throw blockException; + } GroupShardsIterator groups; try { @@ -122,9 +128,9 @@ public abstract class TransportIndexReplicationOperationAction { diff --git a/src/main/java/org/elasticsearch/action/support/replication/TransportIndicesReplicationOperationAction.java b/src/main/java/org/elasticsearch/action/support/replication/TransportIndicesReplicationOperationAction.java index 41705e687a6..251d8ce9073 100644 --- a/src/main/java/org/elasticsearch/action/support/replication/TransportIndicesReplicationOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/replication/TransportIndicesReplicationOperationAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.support.BaseAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; @@ -65,12 +66,16 @@ public abstract class TransportIndicesReplicationOperationAction listener) { ClusterState clusterState = clusterService.state(); - + ClusterBlockException blockException = checkGlobalBlock(clusterState, request); + if (blockException != null) { + throw blockException; + } // get actual indices - String[] concreteIndices = clusterState.metaData().concreteIndices(request.indices()); - - checkBlock(request, concreteIndices, clusterState); + blockException = checkRequestBlock(clusterState, request, concreteIndices); + if (blockException != null) { + throw blockException; + } final AtomicInteger indexCounter = new AtomicInteger(); final AtomicInteger completionCounter = new AtomicInteger(concreteIndices.length); @@ -120,9 +125,9 @@ public abstract class TransportIndicesReplicationOperationAction { diff --git a/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java b/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java index 1524612dc75..95e3b02f969 100644 --- a/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java @@ -28,10 +28,12 @@ import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.TimeoutClusterStateListener; import org.elasticsearch.cluster.action.shard.ShardStateAction; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -42,7 +44,6 @@ import org.elasticsearch.index.IndexShardMissingException; import org.elasticsearch.index.engine.DocumentAlreadyExistsException; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.shard.IllegalIndexShardStateException; -import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.node.NodeClosedException; @@ -128,8 +129,17 @@ public abstract class TransportShardReplicationOperationAction listener) { + request.index(state.metaData().concreteIndex(request.index())); + return true; } protected TransportRequestOptions transportOptions() { @@ -317,12 +327,6 @@ public abstract class TransportShardReplicationOperationAction listener) { this.request = request; this.listener = listener; - - ClusterState clusterState = clusterService.state(); - - nodes = clusterState.nodes(); - - request.index(clusterState.metaData().concreteIndex(request.index())); - - checkBlock(request, clusterState); } public void start() { @@ -128,11 +122,26 @@ public abstract class TransportInstanceSingleOperationAction Date: Fri, 6 Jan 2012 23:41:28 +0200 Subject: [PATCH 125/270] remove yml file conf for test --- .../test/integration/aliases/IndexAliasesTests.java | 11 ++++++++--- .../test/integration/aliases/IndexAliasesTests.yml | 8 -------- 2 files changed, 8 insertions(+), 11 deletions(-) delete mode 100644 src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.yml diff --git a/src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.java b/src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.java index e879539564d..a07fd12b658 100644 --- a/src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.java +++ b/src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.java @@ -29,6 +29,8 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.StopWatch; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.FilterBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -65,11 +67,14 @@ public class IndexAliasesTests extends AbstractNodesTests { protected Client[] clients; protected Random random = new Random(); - @BeforeClass public void startNodes() { - startNode("server1"); - startNode("server2"); + Settings nodeSettings = ImmutableSettings.settingsBuilder() + .put("action.auto_create_index", false) + .build(); + + startNode("server1", nodeSettings); + startNode("server2", nodeSettings); client1 = getClient1(); client2 = getClient2(); clients = new Client[]{client1, client2}; diff --git a/src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.yml b/src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.yml deleted file mode 100644 index 91495fb3cf6..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.yml +++ /dev/null @@ -1,8 +0,0 @@ -cluster: - routing: - schedule: 100ms -index: - number_of_shards: 5 - number_of_replicas: 1 -action: - auto_create_index: false \ No newline at end of file From 164df9979a68c220860d2471ed8699e98a82326b Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 6 Jan 2012 23:43:35 +0200 Subject: [PATCH 126/270] remove yml file conf for test --- .../elasticsearch/test/integration/ping/PingActionTests.yml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 src/test/java/org/elasticsearch/test/integration/ping/PingActionTests.yml diff --git a/src/test/java/org/elasticsearch/test/integration/ping/PingActionTests.yml b/src/test/java/org/elasticsearch/test/integration/ping/PingActionTests.yml deleted file mode 100644 index 641d322a2b3..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/ping/PingActionTests.yml +++ /dev/null @@ -1,6 +0,0 @@ -cluster: - routing: - schedule: 100ms -index: - number_of_shards: 5 - number_of_replicas: 1 From aec5af38000fc48fb481dda453a320499e927a3d Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 7 Jan 2012 00:08:09 +0200 Subject: [PATCH 127/270] clean more test yml files --- .../test/integration/broadcast/BroadcastActionsTests.yml | 7 ------- .../indexlifecycle/IndexLifecycleActionTests.java | 3 +++ .../indexlifecycle/IndexLifecycleActionTests.yml | 6 ------ .../indicesboost/SimpleIndicesBoostSearchTests.java | 9 ++++++++- .../indicesboost/SimpleIndicesBoostSearchTests.yml | 6 ------ .../search/scriptfilter/ScriptFilterSearchTests.java | 9 ++++++++- .../search/scriptfilter/ScriptFilterSearchTests.yml | 6 ------ 7 files changed, 19 insertions(+), 27 deletions(-) delete mode 100644 src/test/java/org/elasticsearch/test/integration/broadcast/BroadcastActionsTests.yml delete mode 100644 src/test/java/org/elasticsearch/test/integration/indexlifecycle/IndexLifecycleActionTests.yml delete mode 100644 src/test/java/org/elasticsearch/test/integration/search/indicesboost/SimpleIndicesBoostSearchTests.yml delete mode 100644 src/test/java/org/elasticsearch/test/integration/search/scriptfilter/ScriptFilterSearchTests.yml diff --git a/src/test/java/org/elasticsearch/test/integration/broadcast/BroadcastActionsTests.yml b/src/test/java/org/elasticsearch/test/integration/broadcast/BroadcastActionsTests.yml deleted file mode 100644 index e6fa6b986d8..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/broadcast/BroadcastActionsTests.yml +++ /dev/null @@ -1,7 +0,0 @@ -cluster: - routing: - schedule: 100ms -index: - number_of_shards: 5 - number_of_replicas: 1 - diff --git a/src/test/java/org/elasticsearch/test/integration/indexlifecycle/IndexLifecycleActionTests.java b/src/test/java/org/elasticsearch/test/integration/indexlifecycle/IndexLifecycleActionTests.java index f959bd370bd..338f7ebc721 100644 --- a/src/test/java/org/elasticsearch/test/integration/indexlifecycle/IndexLifecycleActionTests.java +++ b/src/test/java/org/elasticsearch/test/integration/indexlifecycle/IndexLifecycleActionTests.java @@ -59,6 +59,7 @@ public class IndexLifecycleActionTests extends AbstractNodesTests { Settings settings = settingsBuilder() .put(SETTING_NUMBER_OF_SHARDS, 11) .put(SETTING_NUMBER_OF_REPLICAS, 1) + .put("cluster.routing.schedule", "20ms") // reroute every 20ms so we identify new nodes fast .build(); // start one server @@ -192,6 +193,7 @@ public class IndexLifecycleActionTests extends AbstractNodesTests { Settings settings = settingsBuilder() .put(SETTING_NUMBER_OF_SHARDS, 11) .put(SETTING_NUMBER_OF_REPLICAS, 0) + .put("cluster.routing.schedule", "20ms") // reroute every 20ms so we identify new nodes fast .build(); // start one server @@ -325,6 +327,7 @@ public class IndexLifecycleActionTests extends AbstractNodesTests { Settings settings = settingsBuilder() .put(SETTING_NUMBER_OF_SHARDS, 11) .put(SETTING_NUMBER_OF_REPLICAS, 0) + .put("cluster.routing.schedule", "20ms") // reroute every 20ms so we identify new nodes fast .build(); // start one server diff --git a/src/test/java/org/elasticsearch/test/integration/indexlifecycle/IndexLifecycleActionTests.yml b/src/test/java/org/elasticsearch/test/integration/indexlifecycle/IndexLifecycleActionTests.yml deleted file mode 100644 index 30d98181d38..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/indexlifecycle/IndexLifecycleActionTests.yml +++ /dev/null @@ -1,6 +0,0 @@ -cluster: - routing: - schedule: 100ms -index: - number_of_shards: 11 - number_of_replicas: 1 diff --git a/src/test/java/org/elasticsearch/test/integration/search/indicesboost/SimpleIndicesBoostSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/indicesboost/SimpleIndicesBoostSearchTests.java index 9fabec3f095..c39cb8b2c78 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/indicesboost/SimpleIndicesBoostSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/indicesboost/SimpleIndicesBoostSearchTests.java @@ -22,6 +22,9 @@ package org.elasticsearch.test.integration.search.indicesboost; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.integration.AbstractNodesTests; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -44,7 +47,11 @@ public class SimpleIndicesBoostSearchTests extends AbstractNodesTests { @BeforeMethod public void createNodes() throws Exception { - startNode("server1"); + Settings nodeSettings = ImmutableSettings.settingsBuilder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + startNode("server1", nodeSettings); client = getClient(); } diff --git a/src/test/java/org/elasticsearch/test/integration/search/indicesboost/SimpleIndicesBoostSearchTests.yml b/src/test/java/org/elasticsearch/test/integration/search/indicesboost/SimpleIndicesBoostSearchTests.yml deleted file mode 100644 index e700f1f35fa..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/search/indicesboost/SimpleIndicesBoostSearchTests.yml +++ /dev/null @@ -1,6 +0,0 @@ -cluster: - routing: - schedule: 100ms -index: - number_of_shards: 1 - number_of_replicas: 0 diff --git a/src/test/java/org/elasticsearch/test/integration/search/scriptfilter/ScriptFilterSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/scriptfilter/ScriptFilterSearchTests.java index c2b9646e751..6a5211d2353 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/scriptfilter/ScriptFilterSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/scriptfilter/ScriptFilterSearchTests.java @@ -21,6 +21,9 @@ package org.elasticsearch.test.integration.search.scriptfilter; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.integration.AbstractNodesTests; import org.testng.annotations.AfterMethod; @@ -45,7 +48,11 @@ public class ScriptFilterSearchTests extends AbstractNodesTests { @BeforeMethod public void createNodes() throws Exception { - startNode("server1"); + Settings nodeSettings = ImmutableSettings.settingsBuilder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + startNode("server1", nodeSettings); client = getClient(); } diff --git a/src/test/java/org/elasticsearch/test/integration/search/scriptfilter/ScriptFilterSearchTests.yml b/src/test/java/org/elasticsearch/test/integration/search/scriptfilter/ScriptFilterSearchTests.yml deleted file mode 100644 index e700f1f35fa..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/search/scriptfilter/ScriptFilterSearchTests.yml +++ /dev/null @@ -1,6 +0,0 @@ -cluster: - routing: - schedule: 100ms -index: - number_of_shards: 1 - number_of_replicas: 0 From 3d51553cf270e64d13964d31d21f99d9112341a6 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 7 Jan 2012 23:18:30 +0200 Subject: [PATCH 128/270] Move phonetic token filter to a plugin, closes #1594. --- pom.xml | 12 -- .../index/analysis/AnalysisModule.java | 2 - .../phonetic/DoubleMetaphoneFilter.java | 111 ------------------ .../analysis/phonetic/PhoneticFilter.java | 100 ---------------- .../phonetic/PhoneticTokenFilterFactory.java | 104 ---------------- .../index/analysis/AnalysisModuleTests.java | 9 +- .../test/unit/index/analysis/test1.json | 4 - .../test/unit/index/analysis/test1.yml | 3 - 8 files changed, 1 insertion(+), 344 deletions(-) delete mode 100644 src/main/java/org/elasticsearch/index/analysis/phonetic/DoubleMetaphoneFilter.java delete mode 100644 src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticFilter.java delete mode 100644 src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticTokenFilterFactory.java diff --git a/pom.xml b/pom.xml index 8fa4d30e91e..d759a86378b 100644 --- a/pom.xml +++ b/pom.xml @@ -114,13 +114,6 @@ compile - - commons-codec - commons-codec - 1.6 - compile - - joda-time joda-time @@ -312,7 +305,6 @@ net.sf.trove4j:trove4j org.elasticsearch:es-jsr166y org.elasticsearch:es-jsr166e - commons-codec:commons-codec org.mvel:mvel2 org.codehaus.jackson:jackson-core-asl org.codehaus.jackson:jackson-smile @@ -338,10 +330,6 @@ jsr166e org.elasticsearch.common.util.concurrent.jsr166e - - org.apache.commons.codec - org.elasticsearch.common.codec - org.mvel2 org.elasticsearch.common.mvel2 diff --git a/src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java b/src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java index 7ce840d9cb3..db8c9bf2fb7 100644 --- a/src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java +++ b/src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.settings.NoClassSettingsException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory; import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory; -import org.elasticsearch.index.analysis.phonetic.PhoneticTokenFilterFactory; import org.elasticsearch.indices.analysis.IndicesAnalysisService; import java.util.LinkedList; @@ -452,7 +451,6 @@ public class AnalysisModule extends AbstractModule { tokenFiltersBindings.processTokenFilter("elision", ElisionTokenFilterFactory.class); tokenFiltersBindings.processTokenFilter("pattern_replace", PatternReplaceTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("phonetic", PhoneticTokenFilterFactory.class); tokenFiltersBindings.processTokenFilter("dictionary_decompounder", DictionaryCompoundWordTokenFilterFactory.class); tokenFiltersBindings.processTokenFilter("hyphenation_decompounder", HyphenationCompoundWordTokenFilterFactory.class); diff --git a/src/main/java/org/elasticsearch/index/analysis/phonetic/DoubleMetaphoneFilter.java b/src/main/java/org/elasticsearch/index/analysis/phonetic/DoubleMetaphoneFilter.java deleted file mode 100644 index bd73e509ef9..00000000000 --- a/src/main/java/org/elasticsearch/index/analysis/phonetic/DoubleMetaphoneFilter.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.analysis.phonetic; - -import org.apache.commons.codec.language.DoubleMetaphone; -import org.apache.lucene.analysis.TokenFilter; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; - -import java.io.IOException; -import java.util.LinkedList; - -public final class DoubleMetaphoneFilter extends TokenFilter { - - private static final String TOKEN_TYPE = "DoubleMetaphone"; - - private final LinkedList remainingTokens = new LinkedList(); - private final DoubleMetaphone encoder; - private final boolean inject; - private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); - private final PositionIncrementAttribute posAtt = addAttribute(PositionIncrementAttribute.class); - - public DoubleMetaphoneFilter(TokenStream input, DoubleMetaphone encoder, boolean inject) { - super(input); - this.encoder = encoder; - this.inject = inject; - } - - @Override - public boolean incrementToken() throws IOException { - for (; ; ) { - - if (!remainingTokens.isEmpty()) { - // clearAttributes(); // not currently necessary - restoreState(remainingTokens.removeFirst()); - return true; - } - - if (!input.incrementToken()) return false; - - int len = termAtt.length(); - if (len == 0) return true; // pass through zero length terms - - int firstAlternativeIncrement = inject ? 0 : posAtt.getPositionIncrement(); - - String v = termAtt.toString(); - String primaryPhoneticValue = encoder.doubleMetaphone(v); - String alternatePhoneticValue = encoder.doubleMetaphone(v, true); - - // a flag to lazily save state if needed... this avoids a save/restore when only - // one token will be generated. - boolean saveState = inject; - - if (primaryPhoneticValue != null && primaryPhoneticValue.length() > 0 && !primaryPhoneticValue.equals(v)) { - if (saveState) { - remainingTokens.addLast(captureState()); - } - posAtt.setPositionIncrement(firstAlternativeIncrement); - firstAlternativeIncrement = 0; - termAtt.setEmpty().append(primaryPhoneticValue); - saveState = true; - } - - if (alternatePhoneticValue != null && alternatePhoneticValue.length() > 0 - && !alternatePhoneticValue.equals(primaryPhoneticValue) - && !primaryPhoneticValue.equals(v)) { - if (saveState) { - remainingTokens.addLast(captureState()); - saveState = false; - } - posAtt.setPositionIncrement(firstAlternativeIncrement); - termAtt.setEmpty().append(alternatePhoneticValue); - saveState = true; - } - - // Just one token to return, so no need to capture/restore - // any state, simply return it. - if (remainingTokens.isEmpty()) { - return true; - } - - if (saveState) { - remainingTokens.addLast(captureState()); - } - } - } - - @Override - public void reset() throws IOException { - input.reset(); - remainingTokens.clear(); - } -} diff --git a/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticFilter.java b/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticFilter.java deleted file mode 100644 index d98c66cd11b..00000000000 --- a/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticFilter.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.analysis.phonetic; - -import org.apache.commons.codec.Encoder; -import org.apache.lucene.analysis.TokenFilter; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; - -import java.io.IOException; - -/** - * Create tokens for phonetic matches. See: - * http://jakarta.apache.org/commons/codec/api-release/org/apache/commons/codec/language/package-summary.html - */ -// LUCENE MONITOR -public class PhoneticFilter extends TokenFilter { - - protected boolean inject = true; - protected Encoder encoder = null; - protected String name = null; - - protected State save = null; - private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); - private final PositionIncrementAttribute posAtt = addAttribute(PositionIncrementAttribute.class); - - public PhoneticFilter(TokenStream in, Encoder encoder, String name, boolean inject) { - super(in); - this.encoder = encoder; - this.name = name; - this.inject = inject; - } - - @Override - public boolean incrementToken() throws IOException { - if (save != null) { - // clearAttributes(); // not currently necessary - restoreState(save); - save = null; - return true; - } - - if (!input.incrementToken()) return false; - - // pass through zero-length terms - if (termAtt.length() == 0) return true; - - String value = termAtt.toString(); - String phonetic = null; - try { - String v = encoder.encode(value).toString(); - if (v.length() > 0 && !value.equals(v)) phonetic = v; - } catch (Exception ignored) { - } // just use the direct text - - if (phonetic == null) return true; - - if (!inject) { - // just modify this token - termAtt.setEmpty().append(phonetic); - return true; - } - - // We need to return both the original and the phonetic tokens. - // to avoid a orig=captureState() change_to_phonetic() saved=captureState() restoreState(orig) - // we return the phonetic alternative first - - int origOffset = posAtt.getPositionIncrement(); - posAtt.setPositionIncrement(0); - save = captureState(); - - posAtt.setPositionIncrement(origOffset); - termAtt.setEmpty().append(phonetic); - return true; - } - - @Override - public void reset() throws IOException { - input.reset(); - save = null; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticTokenFilterFactory.java b/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticTokenFilterFactory.java deleted file mode 100644 index b90671f5519..00000000000 --- a/src/main/java/org/elasticsearch/index/analysis/phonetic/PhoneticTokenFilterFactory.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.analysis.phonetic; - -import org.apache.commons.codec.Encoder; -import org.apache.commons.codec.language.*; -import org.apache.commons.codec.language.bm.BeiderMorseEncoder; -import org.apache.commons.codec.language.bm.NameType; -import org.apache.commons.codec.language.bm.RuleType; -import org.apache.lucene.analysis.TokenStream; -import org.elasticsearch.ElasticSearchIllegalArgumentException; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.AnalysisSettingsRequired; -import org.elasticsearch.index.settings.IndexSettings; - -/** - * - */ -@AnalysisSettingsRequired -public class PhoneticTokenFilterFactory extends AbstractTokenFilterFactory { - - private final Encoder encoder; - - private final boolean replace; - - @Inject - public PhoneticTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); - this.replace = settings.getAsBoolean("replace", true); - String encoder = settings.get("encoder"); - if (encoder == null) { - throw new ElasticSearchIllegalArgumentException("encoder must be set on phonetic token filter"); - } - if ("metaphone".equalsIgnoreCase(encoder)) { - this.encoder = new Metaphone(); - } else if ("soundex".equalsIgnoreCase(encoder)) { - this.encoder = new Soundex(); - } else if ("caverphone1".equalsIgnoreCase(encoder)) { - this.encoder = new Caverphone1(); - } else if ("caverphone2".equalsIgnoreCase(encoder)) { - this.encoder = new Caverphone2(); - } else if ("caverphone".equalsIgnoreCase(encoder)) { - this.encoder = new Caverphone2(); - } else if ("refined_soundex".equalsIgnoreCase(encoder) || "refinedSoundex".equalsIgnoreCase(encoder)) { - this.encoder = new RefinedSoundex(); - } else if ("cologne".equalsIgnoreCase(encoder)) { - this.encoder = new ColognePhonetic(); - } else if ("double_metaphone".equalsIgnoreCase(encoder) || "doubleMetaphone".equalsIgnoreCase(encoder)) { - DoubleMetaphone doubleMetaphone = new DoubleMetaphone(); - doubleMetaphone.setMaxCodeLen(settings.getAsInt("max_code_len", doubleMetaphone.getMaxCodeLen())); - this.encoder = doubleMetaphone; - } else if ("bm".equalsIgnoreCase(encoder) || "beider_morse".equalsIgnoreCase(encoder)) { - BeiderMorseEncoder bm = new BeiderMorseEncoder(); - String ruleType = settings.get("rule_type", "approx"); - if ("approx".equalsIgnoreCase(ruleType)) { - bm.setRuleType(RuleType.APPROX); - } else if ("exact".equalsIgnoreCase(ruleType)) { - bm.setRuleType(RuleType.EXACT); - } else { - throw new ElasticSearchIllegalArgumentException("No matching rule type [" + ruleType + "] for beider morse encoder"); - } - String nameType = settings.get("name_type", "generic"); - if ("GENERIC".equalsIgnoreCase(nameType)) { - bm.setNameType(NameType.GENERIC); - } else if ("ASHKENAZI".equalsIgnoreCase(nameType)) { - bm.setNameType(NameType.ASHKENAZI); - } else if ("SEPHARDIC".equalsIgnoreCase(nameType)) { - bm.setNameType(NameType.SEPHARDIC); - } - this.encoder = bm; - } else { - throw new ElasticSearchIllegalArgumentException("unknown encoder [" + encoder + "] for phonetic token filter"); - } - } - - @Override - public TokenStream create(TokenStream tokenStream) { - if (encoder instanceof DoubleMetaphone) { - return new DoubleMetaphoneFilter(tokenStream, (DoubleMetaphone) encoder, !replace); - } - return new PhoneticFilter(tokenStream, encoder, name(), !replace); - } -} \ No newline at end of file diff --git a/src/test/java/org/elasticsearch/test/unit/index/analysis/AnalysisModuleTests.java b/src/test/java/org/elasticsearch/test/unit/index/analysis/AnalysisModuleTests.java index a74d39f5825..165d3748c7f 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/analysis/AnalysisModuleTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/analysis/AnalysisModuleTests.java @@ -32,11 +32,10 @@ import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNameModule; import org.elasticsearch.index.analysis.*; -import org.elasticsearch.test.unit.index.analysis.filter1.MyFilterTokenFilterFactory; -import org.elasticsearch.index.analysis.phonetic.PhoneticTokenFilterFactory; import org.elasticsearch.index.settings.IndexSettingsModule; import org.elasticsearch.indices.analysis.IndicesAnalysisModule; import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.test.unit.index.analysis.filter1.MyFilterTokenFilterFactory; import org.hamcrest.MatcherAssert; import org.testng.annotations.Test; @@ -109,12 +108,6 @@ public class AnalysisModuleTests { analyzer = analysisService.analyzer("alias1").analyzer(); assertThat(analyzer, instanceOf(StandardAnalyzer.class)); - // check phonetic - analyzer = analysisService.analyzer("custom3").analyzer(); - assertThat(analyzer, instanceOf(CustomAnalyzer.class)); - CustomAnalyzer custom3 = (CustomAnalyzer) analyzer; - assertThat(custom3.tokenFilters()[0], instanceOf(PhoneticTokenFilterFactory.class)); - // check custom class name (my) analyzer = analysisService.analyzer("custom4").analyzer(); assertThat(analyzer, instanceOf(CustomAnalyzer.class)); diff --git a/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.json b/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.json index 6b7a8ec67d7..b674552fe2c 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.json +++ b/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.json @@ -53,10 +53,6 @@ "tokenizer":"standard", "char_filter":["html_strip", "my_html"] }, - "custom3":{ - "tokenizer":"standard", - "filter":["metaphone"] - }, "custom4":{ "tokenizer":"standard", "filter":["my"] diff --git a/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.yml b/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.yml index f6c1bc0b0b7..3f4a5c29f38 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.yml +++ b/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.yml @@ -38,9 +38,6 @@ index : custom2 : tokenizer : standard char_filter : [html_strip, my_html] - custom3 : - tokenizer : standard - filter : [metaphone] custom4 : tokenizer : standard filter : [my] From d2e3ff66b2fec780ecc0b3ac43b7decd94335d27 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 11:09:08 +0200 Subject: [PATCH 129/270] netty buffer handling with http always copies over the buffer, so its basically safe --- .../elasticsearch/http/netty/HttpRequestHandler.java | 2 ++ .../elasticsearch/http/netty/NettyHttpRequest.java | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/http/netty/HttpRequestHandler.java b/src/main/java/org/elasticsearch/http/netty/HttpRequestHandler.java index 0f49a2f841a..ed56b361286 100644 --- a/src/main/java/org/elasticsearch/http/netty/HttpRequestHandler.java +++ b/src/main/java/org/elasticsearch/http/netty/HttpRequestHandler.java @@ -38,6 +38,8 @@ public class HttpRequestHandler extends SimpleChannelUpstreamHandler { @Override public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { HttpRequest request = (HttpRequest) e.getMessage(); + // the netty HTTP handling always copy over the buffer to its own buffer, either in NioWorker internally + // when reading, or using a cumalation buffer serverTransport.dispatchRequest(new NettyHttpRequest(request), new NettyHttpChannel(serverTransport, e.getChannel(), request)); super.messageReceived(ctx, e); } diff --git a/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java b/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java index 3951f502467..682d37c5b28 100644 --- a/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java +++ b/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java @@ -19,12 +19,12 @@ package org.elasticsearch.http.netty; +import com.google.common.base.Charsets; import org.elasticsearch.http.HttpRequest; import org.elasticsearch.rest.support.AbstractRestRequest; import org.elasticsearch.rest.support.RestUtils; import org.jboss.netty.handler.codec.http.HttpMethod; -import java.nio.charset.Charset; import java.util.HashMap; import java.util.Map; @@ -108,7 +108,10 @@ public class NettyHttpRequest extends AbstractRestRequest implements HttpRequest @Override public boolean contentUnsafe() { - return request.getContent().hasArray(); + // the netty HTTP handling always copy over the buffer to its own buffer, either in NioWorker internally + // when reading, or using a cumalation buffer + return false; + //return request.getContent().hasArray(); } @Override @@ -133,11 +136,9 @@ public class NettyHttpRequest extends AbstractRestRequest implements HttpRequest return 0; } - private static Charset UTF8 = Charset.forName("UTF-8"); - @Override public String contentAsString() { - return request.getContent().toString(UTF8); + return request.getContent().toString(Charsets.UTF_8); } @Override From e059e213db3bdd8ec7168e8cc4095010b47ed167 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 12:06:30 +0200 Subject: [PATCH 130/270] removed phonetic, fix test config files --- .../org/elasticsearch/test/unit/index/analysis/test1.json | 4 ---- .../java/org/elasticsearch/test/unit/index/analysis/test1.yml | 3 --- 2 files changed, 7 deletions(-) diff --git a/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.json b/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.json index b674552fe2c..9aef36ce09d 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.json +++ b/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.json @@ -26,10 +26,6 @@ "type":"stop", "stopwords":["stop2-1", "stop2-2"] }, - "metaphone":{ - "type":"phonetic", - "encoder":"metaphone" - }, "my":{ "type":"org.elasticsearch.test.unit.index.analysis.filter1.MyFilterTokenFilterFactory" }, diff --git a/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.yml b/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.yml index 3f4a5c29f38..dc31fb8639a 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.yml +++ b/src/test/java/org/elasticsearch/test/unit/index/analysis/test1.yml @@ -18,9 +18,6 @@ index : stop2 : type : stop stopwords : [stop2-1, stop2-2] - metaphone : - type : phonetic - encoder : metaphone my : type : org.elasticsearch.test.unit.index.analysis.filter1.MyFilterTokenFilterFactory dict_dec : From 45b5594e9bc70b694d539718f6d3f972f3d640ab Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 12:17:53 +0200 Subject: [PATCH 131/270] sleep before checking for no master block --- .../test/integration/cluster/NoMasterNodeTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/org/elasticsearch/test/integration/cluster/NoMasterNodeTests.java b/src/test/java/org/elasticsearch/test/integration/cluster/NoMasterNodeTests.java index 94a435161d1..a45b940cb57 100644 --- a/src/test/java/org/elasticsearch/test/integration/cluster/NoMasterNodeTests.java +++ b/src/test/java/org/elasticsearch/test/integration/cluster/NoMasterNodeTests.java @@ -64,6 +64,8 @@ public class NoMasterNodeTests extends AbstractNodesTests { node.client().admin().indices().prepareCreate("test").execute().actionGet(); node2.close(); + Thread.sleep(200); + ClusterState state = node.client().admin().cluster().prepareState().setLocal(true).execute().actionGet().state(); assertThat(state.blocks().hasGlobalBlock(Discovery.NO_MASTER_BLOCK), equalTo(true)); From 0941d157beee9b0c9a321867511baa62d123c916 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 13:13:42 +0200 Subject: [PATCH 132/270] enhance StreamInput to allow to read a bytes ref, and have it directly read it from netty buffers / bytes, apply it to index/percolate/search/count (for now) --- .../action/count/CountRequest.java | 17 +++---- .../action/count/ShardCountRequest.java | 18 ++++---- .../action/index/IndexRequest.java | 26 ++++++----- .../action/percolate/PercolateRequest.java | 22 ++++++--- .../action/search/SearchRequest.java | 41 +++++------------ .../org/elasticsearch/common/BytesHolder.java | 4 +- .../common/io/stream/AdapterStreamInput.java | 7 +++ .../common/io/stream/AdapterStreamOutput.java | 12 +++++ .../common/io/stream/BytesStreamInput.java | 10 +++++ .../common/io/stream/StreamInput.java | 22 +++++++++ .../common/io/stream/StreamOutput.java | 11 +++++ .../internal/InternalSearchRequest.java | 45 ++++++------------- .../netty/ChannelBufferStreamInput.java | 17 ++++++- .../netty/MessageChannelHandler.java | 3 +- 14 files changed, 153 insertions(+), 102 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/count/CountRequest.java b/src/main/java/org/elasticsearch/action/count/CountRequest.java index 4a352aaef8e..bd604fb5ac3 100644 --- a/src/main/java/org/elasticsearch/action/count/CountRequest.java +++ b/src/main/java/org/elasticsearch/action/count/CountRequest.java @@ -25,10 +25,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.broadcast.BroadcastOperationRequest; import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; import org.elasticsearch.client.Requests; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Required; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.Unicode; +import org.elasticsearch.common.*; import org.elasticsearch.common.io.BytesStream; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -48,7 +45,6 @@ import java.util.Map; *

    The request requires the query source to be set either using {@link #query(org.elasticsearch.index.query.QueryBuilder)}, * or {@link #query(byte[])}. * - * * @see CountResponse * @see org.elasticsearch.client.Client#count(CountRequest) * @see org.elasticsearch.client.Requests#countRequest(String...) @@ -292,11 +288,11 @@ public class CountRequest extends BroadcastOperationRequest { routing = in.readUTF(); } + BytesHolder bytes = in.readBytesReference(); querySourceUnsafe = false; - querySourceOffset = 0; - querySourceLength = in.readVInt(); - querySource = new byte[querySourceLength]; - in.readFully(querySource); + querySource = bytes.bytes(); + querySourceOffset = bytes.offset(); + querySourceLength = bytes.length(); int typesSize = in.readVInt(); if (typesSize > 0) { @@ -325,8 +321,7 @@ public class CountRequest extends BroadcastOperationRequest { out.writeUTF(routing); } - out.writeVInt(querySourceLength); - out.writeBytes(querySource, querySourceOffset, querySourceLength); + out.writeBytesHolder(querySource, querySourceOffset, querySourceLength()); out.writeVInt(types.length); for (String type : types) { diff --git a/src/main/java/org/elasticsearch/action/count/ShardCountRequest.java b/src/main/java/org/elasticsearch/action/count/ShardCountRequest.java index 6aa5f7569bf..f0d06d730e7 100644 --- a/src/main/java/org/elasticsearch/action/count/ShardCountRequest.java +++ b/src/main/java/org/elasticsearch/action/count/ShardCountRequest.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.count; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationRequest; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -29,8 +30,6 @@ import java.io.IOException; /** * Internal count request executed directly against a specific index shard. - * - * */ class ShardCountRequest extends BroadcastShardOperationRequest { @@ -87,10 +86,12 @@ class ShardCountRequest extends BroadcastShardOperationRequest { public void readFrom(StreamInput in) throws IOException { super.readFrom(in); minScore = in.readFloat(); - querySourceLength = in.readVInt(); - querySourceOffset = 0; - querySource = new byte[querySourceLength]; - in.readFully(querySource); + + BytesHolder bytes = in.readBytesHolder(); + querySource = bytes.bytes(); + querySourceOffset = bytes.offset(); + querySourceLength = bytes.length(); + int typesSize = in.readVInt(); if (typesSize > 0) { types = new String[typesSize]; @@ -111,8 +112,9 @@ class ShardCountRequest extends BroadcastShardOperationRequest { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeFloat(minScore); - out.writeVInt(querySourceLength); - out.writeBytes(querySource, querySourceOffset, querySourceLength); + + out.writeBytesHolder(querySource, querySourceOffset, querySourceLength); + out.writeVInt(types.length); for (String type : types) { out.writeUTF(type); diff --git a/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/src/main/java/org/elasticsearch/action/index/IndexRequest.java index a65340fea65..a674217b75c 100644 --- a/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -32,10 +32,7 @@ import org.elasticsearch.action.support.replication.ShardReplicationOperationReq import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Required; -import org.elasticsearch.common.UUID; -import org.elasticsearch.common.Unicode; +import org.elasticsearch.common.*; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; @@ -65,7 +62,6 @@ import static org.elasticsearch.action.Actions.addValidationError; *

    *

    If the {@link #id(String)} is not set, it will be automatically generated. * - * * @see IndexResponse * @see org.elasticsearch.client.Requests#indexRequest(String) * @see org.elasticsearch.client.Client#index(IndexRequest) @@ -334,14 +330,23 @@ public class IndexRequest extends ShardReplicationOperationRequest { } public byte[] underlyingSource() { + if (sourceUnsafe) { + source(); + } return this.source; } public int underlyingSourceOffset() { + if (sourceUnsafe) { + source(); + } return this.sourceOffset; } public int underlyingSourceLength() { + if (sourceUnsafe) { + source(); + } return this.sourceLength; } @@ -692,11 +697,11 @@ public class IndexRequest extends ShardReplicationOperationRequest { timestamp = in.readUTF(); } ttl = in.readLong(); + BytesHolder bytes = in.readBytesReference(); sourceUnsafe = false; - sourceOffset = 0; - sourceLength = in.readVInt(); - source = new byte[sourceLength]; - in.readFully(source); + source = bytes.bytes(); + sourceOffset = bytes.offset(); + sourceLength = bytes.length(); opType = OpType.fromId(in.readByte()); refresh = in.readBoolean(); @@ -736,8 +741,7 @@ public class IndexRequest extends ShardReplicationOperationRequest { out.writeUTF(timestamp); } out.writeLong(ttl); - out.writeVInt(sourceLength); - out.writeBytes(source, sourceOffset, sourceLength); + out.writeBytesHolder(source, sourceOffset, sourceLength); out.writeByte(opType.id()); out.writeBoolean(refresh); out.writeLong(version); diff --git a/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java b/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java index 55ba4e9e9fa..cb4916670f8 100644 --- a/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java +++ b/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java @@ -23,6 +23,7 @@ import org.apache.lucene.util.UnicodeUtil; import org.elasticsearch.ElasticSearchGenerationException; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.single.custom.SingleCustomOperationRequest; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Required; import org.elasticsearch.common.Unicode; import org.elasticsearch.common.io.stream.StreamInput; @@ -103,14 +104,23 @@ public class PercolateRequest extends SingleCustomOperationRequest { } public byte[] underlyingSource() { + if (sourceUnsafe) { + source(); + } return this.source; } public int underlyingSourceOffset() { + if (sourceUnsafe) { + source(); + } return this.sourceOffset; } public int underlyingSourceLength() { + if (sourceUnsafe) { + source(); + } return this.sourceLength; } @@ -202,11 +212,11 @@ public class PercolateRequest extends SingleCustomOperationRequest { index = in.readUTF(); type = in.readUTF(); + BytesHolder bytes = in.readBytesReference(); sourceUnsafe = false; - sourceOffset = 0; - sourceLength = in.readVInt(); - source = new byte[sourceLength]; - in.readFully(source); + source = bytes.bytes(); + sourceOffset = bytes.offset(); + sourceLength = bytes.length(); } @Override @@ -214,8 +224,6 @@ public class PercolateRequest extends SingleCustomOperationRequest { super.writeTo(out); out.writeUTF(index); out.writeUTF(type); - - out.writeVInt(sourceLength); - out.writeBytes(source, sourceOffset, sourceLength); + out.writeBytesHolder(source, sourceOffset, sourceLength); } } diff --git a/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/src/main/java/org/elasticsearch/action/search/SearchRequest.java index d621850864e..5dc611a78e6 100644 --- a/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -25,7 +25,7 @@ import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.client.Requests; -import org.elasticsearch.common.Bytes; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.Unicode; @@ -56,7 +56,6 @@ import static org.elasticsearch.search.Scroll.readScroll; *

    *

    There is an option to specify an addition search source using the {@link #extraSource(org.elasticsearch.search.builder.SearchSourceBuilder)}. * - * * @see org.elasticsearch.client.Requests#searchRequest(String...) * @see org.elasticsearch.client.Client#search(SearchRequest) * @see SearchResponse @@ -551,25 +550,17 @@ public class SearchRequest implements ActionRequest { timeout = readTimeValue(in); } + BytesHolder bytes = in.readBytesReference(); sourceUnsafe = false; - sourceOffset = 0; - sourceLength = in.readVInt(); - if (sourceLength == 0) { - source = Bytes.EMPTY_ARRAY; - } else { - source = new byte[sourceLength]; - in.readFully(source); - } + source = bytes.bytes(); + sourceOffset = bytes.offset(); + sourceLength = bytes.length(); + bytes = in.readBytesReference(); extraSourceUnsafe = false; - extraSourceOffset = 0; - extraSourceLength = in.readVInt(); - if (extraSourceLength == 0) { - extraSource = Bytes.EMPTY_ARRAY; - } else { - extraSource = new byte[extraSourceLength]; - in.readFully(extraSource); - } + extraSource = bytes.bytes(); + extraSourceOffset = bytes.offset(); + extraSourceLength = bytes.length(); int typesSize = in.readVInt(); if (typesSize > 0) { @@ -621,18 +612,8 @@ public class SearchRequest implements ActionRequest { out.writeBoolean(true); timeout.writeTo(out); } - if (source == null) { - out.writeVInt(0); - } else { - out.writeVInt(sourceLength); - out.writeBytes(source, sourceOffset, sourceLength); - } - if (extraSource == null) { - out.writeVInt(0); - } else { - out.writeVInt(extraSourceLength); - out.writeBytes(extraSource, extraSourceOffset, extraSourceLength); - } + out.writeBytesHolder(source, sourceOffset, sourceLength); + out.writeBytesHolder(extraSource, extraSourceOffset, extraSourceLength); out.writeVInt(types.length); for (String type : types) { out.writeUTF(type); diff --git a/src/main/java/org/elasticsearch/common/BytesHolder.java b/src/main/java/org/elasticsearch/common/BytesHolder.java index ef646392bb6..c95960c7739 100644 --- a/src/main/java/org/elasticsearch/common/BytesHolder.java +++ b/src/main/java/org/elasticsearch/common/BytesHolder.java @@ -28,6 +28,8 @@ import java.util.Arrays; public class BytesHolder implements Streamable { + public static final BytesHolder EMPTY = new BytesHolder(Bytes.EMPTY_ARRAY, 0, 0); + private byte[] bytes; private int offset; private int length; @@ -75,7 +77,7 @@ public class BytesHolder implements Streamable { offset = 0; length = in.readVInt(); bytes = new byte[length]; - in.readFully(bytes); + in.readBytes(bytes, 0, length); } @Override diff --git a/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamInput.java index c5b61f33b6d..bb1ee572803 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamInput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamInput.java @@ -1,5 +1,7 @@ package org.elasticsearch.common.io.stream; +import org.elasticsearch.common.BytesHolder; + import java.io.IOException; /** @@ -29,6 +31,11 @@ public abstract class AdapterStreamInput extends StreamInput { in.readBytes(b, offset, len); } + @Override + public BytesHolder readBytesReference() throws IOException { + return in.readBytesReference(); + } + @Override public void reset() throws IOException { in.reset(); diff --git a/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java b/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java index 2010160407a..0f9a2ca31b7 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java @@ -19,6 +19,8 @@ package org.elasticsearch.common.io.stream; +import org.elasticsearch.common.BytesHolder; + import java.io.IOException; /** @@ -74,6 +76,16 @@ public class AdapterStreamOutput extends StreamOutput { out.writeBytes(b, length); } + @Override + public void writeBytesHolder(byte[] bytes, int offset, int length) throws IOException { + out.writeBytesHolder(bytes, offset, length); + } + + @Override + public void writeBytesHolder(BytesHolder bytes) throws IOException { + out.writeBytesHolder(bytes); + } + @Override public void writeInt(int i) throws IOException { out.writeInt(i); diff --git a/src/main/java/org/elasticsearch/common/io/stream/BytesStreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/BytesStreamInput.java index 9ecc3a21343..8a4c0d7e75b 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/BytesStreamInput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/BytesStreamInput.java @@ -19,6 +19,8 @@ package org.elasticsearch.common.io.stream; +import org.elasticsearch.common.BytesHolder; + import java.io.EOFException; import java.io.IOException; import java.io.UTFDataFormatException; @@ -44,6 +46,14 @@ public class BytesStreamInput extends StreamInput { this.count = Math.min(offset + length, buf.length); } + @Override + public BytesHolder readBytesReference() throws IOException { + int size = readVInt(); + BytesHolder bytes = new BytesHolder(buf, pos, size); + pos += size; + return bytes; + } + @Override public long skip(long n) throws IOException { if (pos + n > count) { diff --git a/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index 66f2a114b9d..3a3d67af34b 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.io.stream; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Nullable; import java.io.IOException; @@ -51,6 +52,27 @@ public abstract class StreamInput extends InputStream { */ public abstract void readBytes(byte[] b, int offset, int len) throws IOException; + /** + * Reads a fresh copy of the bytes. + */ + public BytesHolder readBytesHolder() throws IOException { + int size = readVInt(); + if (size == 0) { + return BytesHolder.EMPTY; + } + byte[] bytes = new byte[size]; + readBytes(bytes, 0, size); + return new BytesHolder(bytes, 0, size); + } + + /** + * Reads a bytes reference from this stream, might hold an actual reference to the underlying + * bytes of the stream. + */ + public BytesHolder readBytesReference() throws IOException { + return readBytesHolder(); + } + public void readFully(byte[] b) throws IOException { readBytes(b, 0, b.length); } diff --git a/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index 704ab40f469..488ca5727ce 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.io.stream; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Nullable; import java.io.IOException; @@ -70,6 +71,16 @@ public abstract class StreamOutput extends OutputStream { */ public abstract void writeBytes(byte[] b, int offset, int length) throws IOException; + public void writeBytesHolder(byte[] bytes, int offset, int length) throws IOException { + writeVInt(length); + writeBytes(bytes, offset, length); + } + + public void writeBytesHolder(BytesHolder bytes) throws IOException { + writeVInt(bytes.length()); + writeBytes(bytes.bytes(), bytes.offset(), bytes.length()); + } + public final void writeShort(short v) throws IOException { writeByte((byte) (v >> 8)); writeByte((byte) v); diff --git a/src/main/java/org/elasticsearch/search/internal/InternalSearchRequest.java b/src/main/java/org/elasticsearch/search/internal/InternalSearchRequest.java index 20f6013d930..66aef768097 100644 --- a/src/main/java/org/elasticsearch/search/internal/InternalSearchRequest.java +++ b/src/main/java/org/elasticsearch/search/internal/InternalSearchRequest.java @@ -21,7 +21,7 @@ package org.elasticsearch.search.internal; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.common.Bytes; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -50,8 +50,6 @@ import static org.elasticsearch.search.Scroll.readScroll; * } * } * - * - * */ public class InternalSearchRequest implements Streamable { @@ -208,22 +206,17 @@ public class InternalSearchRequest implements Streamable { if (in.readBoolean()) { timeout = readTimeValue(in); } - sourceOffset = 0; - sourceLength = in.readVInt(); - if (sourceLength == 0) { - source = Bytes.EMPTY_ARRAY; - } else { - source = new byte[sourceLength]; - in.readFully(source); - } - extraSourceOffset = 0; - extraSourceLength = in.readVInt(); - if (extraSourceLength == 0) { - extraSource = Bytes.EMPTY_ARRAY; - } else { - extraSource = new byte[extraSourceLength]; - in.readFully(extraSource); - } + + BytesHolder bytes = in.readBytesReference(); + source = bytes.bytes(); + sourceOffset = bytes.offset(); + sourceLength = bytes.length(); + + bytes = in.readBytesReference(); + extraSource = bytes.bytes(); + extraSourceOffset = bytes.offset(); + extraSourceLength = bytes.length(); + int typesSize = in.readVInt(); if (typesSize > 0) { types = new String[typesSize]; @@ -261,18 +254,8 @@ public class InternalSearchRequest implements Streamable { out.writeBoolean(true); timeout.writeTo(out); } - if (source == null) { - out.writeVInt(0); - } else { - out.writeVInt(sourceLength); - out.writeBytes(source, sourceOffset, sourceLength); - } - if (extraSource == null) { - out.writeVInt(0); - } else { - out.writeVInt(extraSourceLength); - out.writeBytes(extraSource, extraSourceOffset, extraSourceLength); - } + out.writeBytesHolder(source, sourceOffset, sourceLength); + out.writeBytesHolder(extraSource, extraSourceOffset, extraSourceLength); out.writeVInt(types.length); for (String type : types) { out.writeUTF(type); diff --git a/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java b/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java index f32d4a70183..8b15a64f019 100644 --- a/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java +++ b/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java @@ -19,6 +19,7 @@ package org.elasticsearch.transport.netty; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.io.stream.StreamInput; import org.jboss.netty.buffer.ChannelBuffer; @@ -27,8 +28,6 @@ import java.io.IOException; /** * A Netty {@link org.jboss.netty.buffer.ChannelBuffer} based {@link org.elasticsearch.common.io.stream.StreamInput}. - * - * */ public class ChannelBufferStreamInput extends StreamInput { @@ -46,6 +45,20 @@ public class ChannelBufferStreamInput extends StreamInput { buffer.markReaderIndex(); } + @Override + public BytesHolder readBytesReference() throws IOException { + // netty always copies a buffer, either in NioWorker in its read handler, where it copies to a fresh + // buffer, or in the cumlation buffer, which is cleaned each time + // so: we can actually return a reference if this is an array backed buffer + if (!buffer.hasArray()) { + return super.readBytesReference(); + } + int size = readVInt(); + BytesHolder bytes = new BytesHolder(buffer.array(), buffer.arrayOffset() + buffer.readerIndex(), size); + buffer.skipBytes(size); + return bytes; + } + /** * Returns the number of read bytes by this stream so far. */ diff --git a/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java b/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java index 7b98998e8da..bd2f1afb8e6 100644 --- a/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java +++ b/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java @@ -137,7 +137,6 @@ public class MessageChannelHandler extends SimpleChannelUpstreamHandler { process(context, channel, cumulation, dataLen); } - // TODO: we can potentially create a cumulation buffer cache, pop/push style if (!cumulation.readable()) { this.cumulation = null; } @@ -180,6 +179,8 @@ public class MessageChannelHandler extends SimpleChannelUpstreamHandler { int markedReaderIndex = buffer.readerIndex(); int expectedIndexReader = markedReaderIndex + size; + // netty always copies a buffer, either in NioWorker in its read handler, where it copies to a fresh + // buffer, or in the cumlation buffer, which is cleaned each time StreamInput streamIn = new ChannelBufferStreamInput(buffer, size); long requestId = buffer.readLong(); From 638a8a19e40964601c37938c2f368cf909dddc67 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 13:21:03 +0200 Subject: [PATCH 133/270] read bytes reference in shard count request --- .../java/org/elasticsearch/action/count/ShardCountRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/action/count/ShardCountRequest.java b/src/main/java/org/elasticsearch/action/count/ShardCountRequest.java index f0d06d730e7..fd322f9a758 100644 --- a/src/main/java/org/elasticsearch/action/count/ShardCountRequest.java +++ b/src/main/java/org/elasticsearch/action/count/ShardCountRequest.java @@ -87,7 +87,7 @@ class ShardCountRequest extends BroadcastShardOperationRequest { super.readFrom(in); minScore = in.readFloat(); - BytesHolder bytes = in.readBytesHolder(); + BytesHolder bytes = in.readBytesReference(); querySource = bytes.bytes(); querySourceOffset = bytes.offset(); querySourceLength = bytes.length(); From 656c3989de79e0367113ddae10ba1032a057c67c Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 13:40:25 +0200 Subject: [PATCH 134/270] read reference bytes in search hit for source so we don't copy it unless needed --- .../common/io/stream/AdapterStreamOutput.java | 3 +- .../common/io/stream/StreamOutput.java | 10 ++++-- .../search/internal/InternalSearchHit.java | 31 ++++++++----------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java b/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java index 0f9a2ca31b7..b5d4940d5b1 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java @@ -20,6 +20,7 @@ package org.elasticsearch.common.io.stream; import org.elasticsearch.common.BytesHolder; +import org.elasticsearch.common.Nullable; import java.io.IOException; @@ -82,7 +83,7 @@ public class AdapterStreamOutput extends StreamOutput { } @Override - public void writeBytesHolder(BytesHolder bytes) throws IOException { + public void writeBytesHolder(@Nullable BytesHolder bytes) throws IOException { out.writeBytesHolder(bytes); } diff --git a/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index 488ca5727ce..53c23f5d2ad 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -76,9 +76,13 @@ public abstract class StreamOutput extends OutputStream { writeBytes(bytes, offset, length); } - public void writeBytesHolder(BytesHolder bytes) throws IOException { - writeVInt(bytes.length()); - writeBytes(bytes.bytes(), bytes.offset(), bytes.length()); + public void writeBytesHolder(@Nullable BytesHolder bytes) throws IOException { + if (bytes == null) { + writeVInt(0); + } else { + writeVInt(bytes.length()); + writeBytes(bytes.bytes(), bytes.offset(), bytes.length()); + } } public final void writeShort(short v) throws IOException { diff --git a/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java b/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java index 543695211b3..46a7bb7050e 100644 --- a/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java +++ b/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.internal; import com.google.common.collect.ImmutableMap; import org.apache.lucene.search.Explanation; import org.elasticsearch.ElasticSearchParseException; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.Unicode; @@ -66,7 +67,7 @@ public class InternalSearchHit implements SearchHit { private long version = -1; - private byte[] source; + private BytesHolder source; private Map fields = ImmutableMap.of(); @@ -91,7 +92,7 @@ public class InternalSearchHit implements SearchHit { this.docId = docId; this.id = id; this.type = type; - this.source = source; + this.source = source == null ? null : new BytesHolder(source); this.fields = fields; } @@ -166,14 +167,14 @@ public class InternalSearchHit implements SearchHit { if (source == null) { return null; } - if (LZF.isCompressed(source)) { + if (LZF.isCompressed(source.bytes(), source.offset(), source.length())) { try { - this.source = LZFDecoder.decode(source); + this.source = new BytesHolder(LZFDecoder.decode(source.bytes(), source.offset(), source.length())); } catch (IOException e) { throw new ElasticSearchParseException("failed to decompress source", e); } } - return this.source; + return this.source.copyBytes(); } @Override @@ -191,7 +192,7 @@ public class InternalSearchHit implements SearchHit { if (source == null) { return null; } - return Unicode.fromBytes(source()); + return Unicode.fromBytes(source.bytes(), source.offset(), source.length()); } @SuppressWarnings({"unchecked"}) @@ -362,7 +363,7 @@ public class InternalSearchHit implements SearchHit { builder.field(Fields._SCORE, score); } if (source != null) { - RestXContentBuilder.restDocumentSource(source, builder, params); + RestXContentBuilder.restDocumentSource(source.bytes(), source.offset(), source.length(), builder, params); } if (fields != null && !fields.isEmpty()) { builder.startObject(Fields.FIELDS); @@ -452,15 +453,14 @@ public class InternalSearchHit implements SearchHit { id = in.readUTF(); type = in.readUTF(); version = in.readLong(); - int size = in.readVInt(); - if (size > 0) { - source = new byte[size]; - in.readFully(source); + source = in.readBytesReference(); + if (source.length() == 0) { + source = null; } if (in.readBoolean()) { explanation = readExplanation(in); } - size = in.readVInt(); + int size = in.readVInt(); if (size == 0) { fields = ImmutableMap.of(); } else if (size == 1) { @@ -586,12 +586,7 @@ public class InternalSearchHit implements SearchHit { out.writeUTF(id); out.writeUTF(type); out.writeLong(version); - if (source == null) { - out.writeVInt(0); - } else { - out.writeVInt(source.length); - out.writeBytes(source); - } + out.writeBytesHolder(source); if (explanation == null) { out.writeBoolean(false); } else { From 3cd08e49e8b7d31800aa1627d3d95e3d7446fab8 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 13:51:57 +0200 Subject: [PATCH 135/270] clean unused method --- .../transport/netty/ChannelBufferStreamInput.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java b/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java index 8b15a64f019..5f660526142 100644 --- a/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java +++ b/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java @@ -59,13 +59,6 @@ public class ChannelBufferStreamInput extends StreamInput { return bytes; } - /** - * Returns the number of read bytes by this stream so far. - */ - public int readBytes() { - return buffer.readerIndex() - startIndex; - } - @Override public int available() throws IOException { return endIndex - buffer.readerIndex(); From ef9c96faa66dec497f4e9621a3bd8b707509ebe3 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 13:57:10 +0200 Subject: [PATCH 136/270] just read byte in channel buffer wrapper, no need to check for bounds since it is checked internally --- .../transport/netty/ChannelBufferStreamInput.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java b/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java index 5f660526142..5edca5b1e07 100644 --- a/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java +++ b/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java @@ -120,9 +120,6 @@ public class ChannelBufferStreamInput extends StreamInput { @Override public byte readByte() throws IOException { - if (available() == 0) { - throw new EOFException(); - } return buffer.readByte(); } From 7966716673f3ea5858eae560f11dd598dbff6a0d Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 15:26:33 +0200 Subject: [PATCH 137/270] Improve serialization (stream) of UTF strings, note, requires flush when upgrading, closes #1595. --- .../common/io/stream/BytesStreamInput.java | 79 ------------------- .../common/io/stream/BytesStreamOutput.java | 58 -------------- .../common/io/stream/CachedStreamInput.java | 9 +++ .../common/io/stream/StreamInput.java | 77 +++--------------- .../common/io/stream/StreamOutput.java | 60 +++----------- 5 files changed, 29 insertions(+), 254 deletions(-) diff --git a/src/main/java/org/elasticsearch/common/io/stream/BytesStreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/BytesStreamInput.java index 8a4c0d7e75b..19e9574c904 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/BytesStreamInput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/BytesStreamInput.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.BytesHolder; import java.io.EOFException; import java.io.IOException; -import java.io.UTFDataFormatException; /** * @@ -135,82 +134,4 @@ public class BytesStreamInput extends StreamInput { public void close() throws IOException { // nothing to do here... } - - public String readUTF() throws IOException { - int utflen = readInt(); - if (utflen == 0) { - return ""; - } - if (chararr.length < utflen) { - chararr = new char[utflen * 2]; - } - char[] chararr = this.chararr; - byte[] bytearr = buf; - int endPos = pos + utflen; - - int c, char2, char3; - int count = pos; - int chararr_count = 0; - - while (count < endPos) { - c = (int) bytearr[count] & 0xff; - if (c > 127) break; - count++; - chararr[chararr_count++] = (char) c; - } - - while (count < endPos) { - c = (int) bytearr[count] & 0xff; - switch (c >> 4) { - case 0: - case 1: - case 2: - case 3: - case 4: - case 5: - case 6: - case 7: - /* 0xxxxxxx*/ - count++; - chararr[chararr_count++] = (char) c; - break; - case 12: - case 13: - /* 110x xxxx 10xx xxxx*/ - count += 2; - if (count > endPos) - throw new UTFDataFormatException( - "malformed input: partial character at end"); - char2 = (int) bytearr[count - 1]; - if ((char2 & 0xC0) != 0x80) - throw new UTFDataFormatException( - "malformed input around byte " + count); - chararr[chararr_count++] = (char) (((c & 0x1F) << 6) | - (char2 & 0x3F)); - break; - case 14: - /* 1110 xxxx 10xx xxxx 10xx xxxx */ - count += 3; - if (count > endPos) - throw new UTFDataFormatException( - "malformed input: partial character at end"); - char2 = (int) bytearr[count - 2]; - char3 = (int) bytearr[count - 1]; - if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) - throw new UTFDataFormatException( - "malformed input around byte " + (count - 1)); - chararr[chararr_count++] = (char) (((c & 0x0F) << 12) | - ((char2 & 0x3F) << 6) | - ((char3 & 0x3F) << 0)); - break; - default: - /* 10xx xxxx, 1111 xxxx */ - throw new UTFDataFormatException( - "malformed input around byte " + count); - } - } - pos += utflen; - // The number of chars produced may be less than utflen - return new String(chararr, 0, chararr_count); - } } diff --git a/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java b/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java index 074e0769662..06d77ae01b0 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java @@ -118,62 +118,4 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream { public int size() { return count; } - - - /** - * Writes a string. - */ - // Override here since we can work on the byte array directly! - public void writeUTF(String str) throws IOException { - int strlen = str.length(); - int utflen = 0; - int c = 0; - - /* use charAt instead of copying String to char array */ - for (int i = 0; i < strlen; i++) { - c = str.charAt(i); - if ((c >= 0x0001) && (c <= 0x007F)) { - utflen++; - } else if (c > 0x07FF) { - utflen += 3; - } else { - utflen += 2; - } - } - - int newcount = count + utflen + 4; - if (newcount > buf.length) { - buf = Arrays.copyOf(buf, Math.max(buf.length << 1, newcount)); - } - - byte[] bytearr = this.buf; - - // same as writeInt - bytearr[count++] = (byte) (utflen >> 24); - bytearr[count++] = (byte) (utflen >> 16); - bytearr[count++] = (byte) (utflen >> 8); - bytearr[count++] = (byte) (utflen); - - int i = 0; - for (i = 0; i < strlen; i++) { - c = str.charAt(i); - if (!((c >= 0x0001) && (c <= 0x007F))) break; - bytearr[count++] = (byte) c; - } - - for (; i < strlen; i++) { - c = str.charAt(i); - if ((c >= 0x0001) && (c <= 0x007F)) { - bytearr[count++] = (byte) c; - - } else if (c > 0x07FF) { - bytearr[count++] = (byte) (0xE0 | ((c >> 12) & 0x0F)); - bytearr[count++] = (byte) (0x80 | ((c >> 6) & 0x3F)); - bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F)); - } else { - bytearr[count++] = (byte) (0xC0 | ((c >> 6) & 0x1F)); - bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F)); - } - } - } } diff --git a/src/main/java/org/elasticsearch/common/io/stream/CachedStreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/CachedStreamInput.java index 9f4dfbeb338..ef5d867608c 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/CachedStreamInput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/CachedStreamInput.java @@ -28,6 +28,7 @@ import java.lang.ref.SoftReference; public class CachedStreamInput { static class Entry { + char[] chars = new char[80]; final HandlesStreamInput handles; final LZFStreamInput lzf; @@ -73,4 +74,12 @@ public class CachedStreamInput { entry.handles.reset(entry.lzf); return entry.handles; } + + public static char[] getCharArray(int size) { + Entry entry = instance(); + if (entry.chars.length < size) { + entry.chars = new char[size]; + } + return entry.chars; + } } diff --git a/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index 3a3d67af34b..4d3f15bedc3 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.Nullable; import java.io.IOException; import java.io.InputStream; -import java.io.UTFDataFormatException; import java.util.*; /** @@ -32,12 +31,6 @@ import java.util.*; */ public abstract class StreamInput extends InputStream { - /** - * working arrays initialized on demand by readUTF - */ - private byte bytearr[] = new byte[80]; - protected char chararr[] = new char[80]; - /** * Reads and returns a single byte. */ @@ -154,35 +147,12 @@ public abstract class StreamInput extends InputStream { return i | ((b & 0x7FL) << 56); } - // COPIED from DataInputStream - public String readUTF() throws IOException { - int utflen = readInt(); - if (utflen == 0) { - return ""; - } - if (bytearr.length < utflen) { - bytearr = new byte[utflen * 2]; - chararr = new char[utflen * 2]; - } - char[] chararr = this.chararr; - byte[] bytearr = this.bytearr; - - int c, char2, char3; - int count = 0; - int chararr_count = 0; - - readBytes(bytearr, 0, utflen); - - while (count < utflen) { - c = (int) bytearr[count] & 0xff; - if (c > 127) break; - count++; - chararr[chararr_count++] = (char) c; - } - - while (count < utflen) { - c = (int) bytearr[count] & 0xff; + int charCount = readVInt(); + char[] chars = CachedStreamInput.getCharArray(charCount); + int c, charIndex = 0; + while (charIndex < charCount) { + c = readByte() & 0xff; switch (c >> 4) { case 0: case 1: @@ -192,47 +162,18 @@ public abstract class StreamInput extends InputStream { case 5: case 6: case 7: - /* 0xxxxxxx*/ - count++; - chararr[chararr_count++] = (char) c; + chars[charIndex++] = (char) c; break; case 12: case 13: - /* 110x xxxx 10xx xxxx*/ - count += 2; - if (count > utflen) - throw new UTFDataFormatException( - "malformed input: partial character at end"); - char2 = (int) bytearr[count - 1]; - if ((char2 & 0xC0) != 0x80) - throw new UTFDataFormatException( - "malformed input around byte " + count); - chararr[chararr_count++] = (char) (((c & 0x1F) << 6) | - (char2 & 0x3F)); + chars[charIndex++] = (char) ((c & 0x1F) << 6 | readByte() & 0x3F); break; case 14: - /* 1110 xxxx 10xx xxxx 10xx xxxx */ - count += 3; - if (count > utflen) - throw new UTFDataFormatException( - "malformed input: partial character at end"); - char2 = (int) bytearr[count - 2]; - char3 = (int) bytearr[count - 1]; - if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) - throw new UTFDataFormatException( - "malformed input around byte " + (count - 1)); - chararr[chararr_count++] = (char) (((c & 0x0F) << 12) | - ((char2 & 0x3F) << 6) | - ((char3 & 0x3F) << 0)); + chars[charIndex++] = (char) ((c & 0x0F) << 12 | (readByte() & 0x3F) << 6 | (readByte() & 0x3F) << 0); break; - default: - /* 10xx xxxx, 1111 xxxx */ - throw new UTFDataFormatException( - "malformed input around byte " + count); } } - // The number of chars produced may be less than utflen - return new String(chararr, 0, chararr_count); + return new String(chars, 0, charCount); } diff --git a/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index 53c23f5d2ad..9dccd66b86b 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -33,11 +33,6 @@ import java.util.Map; */ public abstract class StreamOutput extends OutputStream { - /** - * bytearr is initialized on demand by writeUTF - */ - private byte[] bytearr = null; - /** * Writes a single byte. */ @@ -138,55 +133,22 @@ public abstract class StreamOutput extends OutputStream { * Writes a string. */ public void writeUTF(String str) throws IOException { - int strlen = str.length(); - int utflen = 0; - int c, count = 0; - - /* use charAt instead of copying String to char array */ - for (int i = 0; i < strlen; i++) { + int charCount = str.length(); + writeVInt(charCount); + int c; + for (int i = 0; i < charCount; i++) { c = str.charAt(i); - if ((c >= 0x0001) && (c <= 0x007F)) { - utflen++; + if (c <= 0x007F) { + writeByte((byte) c); } else if (c > 0x07FF) { - utflen += 3; + writeByte((byte) (0xE0 | c >> 12 & 0x0F)); + writeByte((byte) (0x80 | c >> 6 & 0x3F)); + writeByte((byte) (0x80 | c >> 0 & 0x3F)); } else { - utflen += 2; + writeByte((byte) (0xC0 | c >> 6 & 0x1F)); + writeByte((byte) (0x80 | c >> 0 & 0x3F)); } } - - if (this.bytearr == null || (this.bytearr.length < (utflen + 4))) - this.bytearr = new byte[(utflen * 2) + 4]; - byte[] bytearr = this.bytearr; - - // same as writeInt - bytearr[count++] = (byte) (utflen >> 24); - bytearr[count++] = (byte) (utflen >> 16); - bytearr[count++] = (byte) (utflen >> 8); - bytearr[count++] = (byte) (utflen); - - int i = 0; - for (i = 0; i < strlen; i++) { - c = str.charAt(i); - if (!((c >= 0x0001) && (c <= 0x007F))) break; - bytearr[count++] = (byte) c; - } - - for (; i < strlen; i++) { - c = str.charAt(i); - if ((c >= 0x0001) && (c <= 0x007F)) { - bytearr[count++] = (byte) c; - - } else if (c > 0x07FF) { - bytearr[count++] = (byte) (0xE0 | ((c >> 12) & 0x0F)); - bytearr[count++] = (byte) (0x80 | ((c >> 6) & 0x3F)); - bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F)); - } else { - bytearr[count++] = (byte) (0xC0 | ((c >> 6) & 0x1F)); - bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F)); - } - } - writeBytes(bytearr, 0, utflen + 4); -// return utflen + 2; } public void writeFloat(float v) throws IOException { From cf73e181460703690a8e7728626b67593b5e8977 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 15:47:32 +0200 Subject: [PATCH 138/270] get / mget to read byte reference for the source --- .../org/elasticsearch/common/BytesHolder.java | 27 +------------------ .../elasticsearch/index/get/GetResult.java | 15 ++++------- 2 files changed, 6 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/elasticsearch/common/BytesHolder.java b/src/main/java/org/elasticsearch/common/BytesHolder.java index c95960c7739..3875f832768 100644 --- a/src/main/java/org/elasticsearch/common/BytesHolder.java +++ b/src/main/java/org/elasticsearch/common/BytesHolder.java @@ -19,14 +19,9 @@ package org.elasticsearch.common; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; - -import java.io.IOException; import java.util.Arrays; -public class BytesHolder implements Streamable { +public class BytesHolder { public static final BytesHolder EMPTY = new BytesHolder(Bytes.EMPTY_ARRAY, 0, 0); @@ -66,26 +61,6 @@ public class BytesHolder implements Streamable { return length; } - public static BytesHolder readBytesHolder(StreamInput in) throws IOException { - BytesHolder holder = new BytesHolder(); - holder.readFrom(in); - return holder; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - offset = 0; - length = in.readVInt(); - bytes = new byte[length]; - in.readBytes(bytes, 0, length); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(length); - out.writeBytes(bytes, offset, length); - } - @Override public boolean equals(Object obj) { return bytesEquals((BytesHolder) obj); diff --git a/src/main/java/org/elasticsearch/index/get/GetResult.java b/src/main/java/org/elasticsearch/index/get/GetResult.java index ef15d4089be..b8d9f8f404e 100644 --- a/src/main/java/org/elasticsearch/index/get/GetResult.java +++ b/src/main/java/org/elasticsearch/index/get/GetResult.java @@ -170,8 +170,7 @@ public class GetResult implements Streamable, Iterable, ToXContent { public BytesHolder sourceRef() { if (LZF.isCompressed(source.bytes(), source.offset(), source.length())) { try { - // TODO decompress without doing an extra copy! - this.source = new BytesHolder(LZFDecoder.decode(source.copyBytes())); + this.source = new BytesHolder(LZFDecoder.decode(source.bytes(), source.offset(), source.length())); } catch (IOException e) { throw new ElasticSearchParseException("failed to decompress source", e); } @@ -315,8 +314,9 @@ public class GetResult implements Streamable, Iterable, ToXContent { version = in.readLong(); exists = in.readBoolean(); if (exists) { - if (in.readBoolean()) { - source = BytesHolder.readBytesHolder(in); + source = in.readBytesReference(); + if (source.length() == 0) { + source = null; } int size = in.readVInt(); if (size == 0) { @@ -339,12 +339,7 @@ public class GetResult implements Streamable, Iterable, ToXContent { out.writeLong(version); out.writeBoolean(exists); if (exists) { - if (source == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - source.writeTo(out); - } + out.writeBytesHolder(source); if (fields == null) { out.writeVInt(0); } else { From c02dc8f4f80139a38a4ec5cedffa84974904085b Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 16:41:24 +0200 Subject: [PATCH 139/270] recovery chunk to use bytes ref --- .../recovery/RecoveryFileChunkRequest.java | 20 ++++++------------- .../indices/recovery/RecoverySource.java | 4 +++- .../indices/recovery/RecoveryTarget.java | 8 +++----- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/elasticsearch/indices/recovery/RecoveryFileChunkRequest.java b/src/main/java/org/elasticsearch/indices/recovery/RecoveryFileChunkRequest.java index da6cb17d950..e3ecd7b8103 100644 --- a/src/main/java/org/elasticsearch/indices/recovery/RecoveryFileChunkRequest.java +++ b/src/main/java/org/elasticsearch/indices/recovery/RecoveryFileChunkRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.indices.recovery; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -37,20 +38,18 @@ class RecoveryFileChunkRequest implements Streamable { private long position; private long length; private String checksum; - private byte[] content; - private int contentLength; + private BytesHolder content; RecoveryFileChunkRequest() { } - RecoveryFileChunkRequest(ShardId shardId, String name, long position, long length, String checksum, byte[] content, int contentLength) { + RecoveryFileChunkRequest(ShardId shardId, String name, long position, long length, String checksum, BytesHolder content) { this.shardId = shardId; this.name = name; this.position = position; this.length = length; this.checksum = checksum; this.content = content; - this.contentLength = contentLength; } public ShardId shardId() { @@ -74,14 +73,10 @@ class RecoveryFileChunkRequest implements Streamable { return length; } - public byte[] content() { + public BytesHolder content() { return content; } - public int contentLength() { - return contentLength; - } - public RecoveryFileChunkRequest readFileChunk(StreamInput in) throws IOException { RecoveryFileChunkRequest request = new RecoveryFileChunkRequest(); request.readFrom(in); @@ -97,9 +92,7 @@ class RecoveryFileChunkRequest implements Streamable { if (in.readBoolean()) { checksum = in.readUTF(); } - contentLength = in.readVInt(); - content = new byte[contentLength]; - in.readFully(content); + content = in.readBytesReference(); } @Override @@ -114,8 +107,7 @@ class RecoveryFileChunkRequest implements Streamable { out.writeBoolean(true); out.writeUTF(checksum); } - out.writeVInt(contentLength); - out.writeBytes(content, 0, contentLength); + out.writeBytesHolder(content); } @Override diff --git a/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java b/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java index 67762296cf6..b7028026695 100644 --- a/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java +++ b/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java @@ -23,6 +23,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.lucene.store.IndexInput; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.StopWatch; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; @@ -148,7 +149,8 @@ public class RecoverySource extends AbstractComponent { } indexInput.readBytes(buf, 0, toRead, false); - transportService.submitRequest(request.targetNode(), RecoveryTarget.Actions.FILE_CHUNK, new RecoveryFileChunkRequest(request.shardId(), name, position, len, md.checksum(), buf, toRead), + BytesHolder content = new BytesHolder(buf, 0, toRead); + transportService.submitRequest(request.targetNode(), RecoveryTarget.Actions.FILE_CHUNK, new RecoveryFileChunkRequest(request.shardId(), name, position, len, md.checksum(), content), TransportRequestOptions.options().withCompress(recoverySettings.compress()).withLowType(), VoidTransportResponseHandler.INSTANCE_SAME).txGet(); readCount += toRead; } diff --git a/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java index a2e6e9ac55d..1ce4036377d 100644 --- a/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java +++ b/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java @@ -59,8 +59,6 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; *

    *

    Note, it can be safely assumed that there will only be a single recovery per shard (index+id) and * not several of them (since we don't allocate several shard replicas to the same node). - * - * */ public class RecoveryTarget extends AbstractComponent { @@ -534,10 +532,10 @@ public class RecoveryTarget extends AbstractComponent { synchronized (indexOutput) { try { if (recoverySettings.rateLimiter() != null) { - recoverySettings.rateLimiter().pause(request.contentLength()); + recoverySettings.rateLimiter().pause(request.content().length()); } - indexOutput.writeBytes(request.content(), request.contentLength()); - onGoingRecovery.currentFilesSize.addAndGet(request.contentLength()); + indexOutput.writeBytes(request.content().bytes(), request.content().offset(), request.content().length()); + onGoingRecovery.currentFilesSize.addAndGet(request.length()); if (indexOutput.getFilePointer() == request.length()) { // we are done indexOutput.close(); From 858195351b03a1101cef26825e89913d534b06de Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 17:23:37 +0200 Subject: [PATCH 140/270] translog actions to use bytes ref serialization, and have the option to mark BytesStreamInput as unsafe --- .../elasticsearch/cluster/ClusterState.java | 2 +- .../common/io/stream/BytesStreamInput.java | 12 +- .../common/xcontent/XContentHelper.java | 4 +- .../zen/ping/multicast/MulticastZenPing.java | 2 +- .../publish/PublishClusterStateAction.java | 4 +- .../gateway/blobstore/BlobStoreGateway.java | 2 +- .../gateway/local/LocalGateway.java | 4 +- .../blobstore/BlobStoreIndexShardGateway.java | 2 +- .../shard/service/InternalIndexShard.java | 4 +- .../index/translog/Translog.java | 131 +++--------------- .../index/translog/TranslogStreams.java | 2 +- .../index/translog/fs/FsChannelSnapshot.java | 2 +- .../action/support/RestXContentBuilder.java | 2 +- .../transport/local/LocalTransport.java | 2 +- .../ClusterSerializationTests.java | 2 +- .../common/io/streams/BytesStreamsTests.java | 2 +- .../io/streams/HandlesStreamsTests.java | 2 +- .../engine/AbstractSimpleEngineTests.java | 10 +- .../translog/AbstractSimpleTranslogTests.java | 10 +- 19 files changed, 59 insertions(+), 142 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/ClusterState.java b/src/main/java/org/elasticsearch/cluster/ClusterState.java index fb17bc55fff..69eecccc10c 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -233,7 +233,7 @@ public class ClusterState { } public static ClusterState fromBytes(byte[] data, DiscoveryNode localNode) throws IOException { - return readFrom(new BytesStreamInput(data), localNode); + return readFrom(new BytesStreamInput(data, false), localNode); } public static void writeTo(ClusterState state, StreamOutput out) throws IOException { diff --git a/src/main/java/org/elasticsearch/common/io/stream/BytesStreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/BytesStreamInput.java index 19e9574c904..7177ea36991 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/BytesStreamInput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/BytesStreamInput.java @@ -35,18 +35,24 @@ public class BytesStreamInput extends StreamInput { protected int count; - public BytesStreamInput(byte buf[]) { - this(buf, 0, buf.length); + private final boolean unsafe; + + public BytesStreamInput(byte buf[], boolean unsafe) { + this(buf, 0, buf.length, unsafe); } - public BytesStreamInput(byte buf[], int offset, int length) { + public BytesStreamInput(byte buf[], int offset, int length, boolean unsafe) { this.buf = buf; this.pos = offset; this.count = Math.min(offset + length, buf.length); + this.unsafe = unsafe; } @Override public BytesHolder readBytesReference() throws IOException { + if (unsafe) { + return readBytesHolder(); + } int size = readVInt(); BytesHolder bytes = new BytesHolder(buf, pos, size); pos += size; diff --git a/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 75b70983afd..e170f66c6a3 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -40,7 +40,7 @@ public class XContentHelper { public static XContentParser createParser(byte[] data, int offset, int length) throws IOException { if (LZF.isCompressed(data, offset, length)) { - BytesStreamInput siBytes = new BytesStreamInput(data, offset, length); + BytesStreamInput siBytes = new BytesStreamInput(data, offset, length, false); LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes); XContentType contentType = XContentFactory.xContentType(siLzf); siLzf.resetToBufferStart(); @@ -55,7 +55,7 @@ public class XContentHelper { XContentParser parser; XContentType contentType; if (LZF.isCompressed(data, offset, length)) { - BytesStreamInput siBytes = new BytesStreamInput(data, offset, length); + BytesStreamInput siBytes = new BytesStreamInput(data, offset, length, false); LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes); contentType = XContentFactory.xContentType(siLzf); siLzf.resetToBufferStart(); diff --git a/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java b/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java index 9c042216cf4..2693049d7b3 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java @@ -396,7 +396,7 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem } } if (internal) { - StreamInput input = CachedStreamInput.cachedHandles(new BytesStreamInput(datagramPacketReceive.getData(), datagramPacketReceive.getOffset() + INTERNAL_HEADER.length, datagramPacketReceive.getLength())); + StreamInput input = CachedStreamInput.cachedHandles(new BytesStreamInput(datagramPacketReceive.getData(), datagramPacketReceive.getOffset() + INTERNAL_HEADER.length, datagramPacketReceive.getLength(), true)); Version version = Version.readVersion(input); id = input.readInt(); clusterName = ClusterName.readClusterName(input); diff --git a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java index db1ef3f1dd3..51970a98886 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java +++ b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java @@ -95,7 +95,7 @@ public class PublishClusterStateAction extends AbstractComponent { } } - private class PublishClusterStateRequest implements Streamable { + class PublishClusterStateRequest implements Streamable { private byte[] clusterStateInBytes; @@ -130,7 +130,7 @@ public class PublishClusterStateAction extends AbstractComponent { @Override public void messageReceived(PublishClusterStateRequest request, TransportChannel channel) throws Exception { - StreamInput in = CachedStreamInput.cachedHandlesLzf(new BytesStreamInput(request.clusterStateInBytes)); + StreamInput in = CachedStreamInput.cachedHandlesLzf(new BytesStreamInput(request.clusterStateInBytes, false)); ClusterState clusterState = ClusterState.Builder.readFrom(in, nodesProvider.nodes().localNode()); listener.onNewClusterState(clusterState); channel.sendResponse(VoidStreamable.INSTANCE); diff --git a/src/main/java/org/elasticsearch/gateway/blobstore/BlobStoreGateway.java b/src/main/java/org/elasticsearch/gateway/blobstore/BlobStoreGateway.java index 23463e7ccf6..07edbca12e8 100644 --- a/src/main/java/org/elasticsearch/gateway/blobstore/BlobStoreGateway.java +++ b/src/main/java/org/elasticsearch/gateway/blobstore/BlobStoreGateway.java @@ -207,7 +207,7 @@ public abstract class BlobStoreGateway extends SharedStorageGateway { XContentParser parser = null; try { if (LZF.isCompressed(data)) { - BytesStreamInput siBytes = new BytesStreamInput(data); + BytesStreamInput siBytes = new BytesStreamInput(data, false); LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes); parser = XContentFactory.xContent(XContentType.JSON).createParser(siLzf); } else { diff --git a/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java b/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java index f7fc9cf2d73..1fae23024c0 100644 --- a/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java +++ b/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java @@ -374,7 +374,7 @@ public class LocalGateway extends AbstractLifecycleComponent implements XContentParser parser = null; try { if (LZF.isCompressed(data)) { - BytesStreamInput siBytes = new BytesStreamInput(data); + BytesStreamInput siBytes = new BytesStreamInput(data, false); LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes); parser = XContentFactory.xContent(XContentType.JSON).createParser(siLzf); } else { @@ -392,7 +392,7 @@ public class LocalGateway extends AbstractLifecycleComponent implements XContentParser parser = null; try { if (LZF.isCompressed(data)) { - BytesStreamInput siBytes = new BytesStreamInput(data); + BytesStreamInput siBytes = new BytesStreamInput(data, false); LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes); parser = XContentFactory.xContent(XContentType.JSON).createParser(siLzf); } else { diff --git a/src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java b/src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java index b6de2744636..45fcb2af431 100644 --- a/src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java +++ b/src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java @@ -462,7 +462,7 @@ public abstract class BlobStoreIndexShardGateway extends AbstractIndexShardCompo if (bos.size() < 4) { return; } - BytesStreamInput si = new BytesStreamInput(bos.underlyingBytes(), 0, bos.size()); + BytesStreamInput si = new BytesStreamInput(bos.underlyingBytes(), 0, bos.size(), false); int position; while (true) { try { diff --git a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java index ecca0b93068..24e70962431 100644 --- a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java +++ b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java @@ -596,13 +596,13 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I switch (operation.opType()) { case CREATE: Translog.Create create = (Translog.Create) operation; - engine.create(prepareCreate(source(create.source(), create.sourceOffset(), create.sourceLength()).type(create.type()).id(create.id()) + engine.create(prepareCreate(source(create.source().bytes(), create.source().offset(), create.source().length()).type(create.type()).id(create.id()) .routing(create.routing()).parent(create.parent()).timestamp(create.timestamp()).ttl(create.ttl())).version(create.version()) .origin(Engine.Operation.Origin.RECOVERY)); break; case SAVE: Translog.Index index = (Translog.Index) operation; - engine.index(prepareIndex(source(index.source(), index.sourceOffset(), index.sourceLength()).type(index.type()).id(index.id()) + engine.index(prepareIndex(source(index.source().bytes(), index.source().offset(), index.source().length()).type(index.type()).id(index.id()) .routing(index.routing()).parent(index.parent()).timestamp(index.timestamp()).ttl(index.ttl())).version(index.version()) .origin(Engine.Operation.Origin.RECOVERY)); break; diff --git a/src/main/java/org/elasticsearch/index/translog/Translog.java b/src/main/java/org/elasticsearch/index/translog/Translog.java index 6fe28e42511..0907695ec89 100644 --- a/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -24,7 +24,6 @@ import org.elasticsearch.ElasticSearchIllegalStateException; import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.BytesStreamInput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -219,7 +218,7 @@ public interface Translog extends IndexShardComponent { long estimateSize(); - Source readSource(BytesStreamInput in) throws IOException; + Source readSource(StreamInput in) throws IOException; } static class Source { @@ -241,9 +240,7 @@ public interface Translog extends IndexShardComponent { static class Create implements Operation { private String id; private String type; - private byte[] source; - private int sourceOffset; - private int sourceLength; + private BytesHolder source; private String routing; private String parent; private long timestamp; @@ -256,9 +253,7 @@ public interface Translog extends IndexShardComponent { public Create(Engine.Create create) { this.id = create.id(); this.type = create.type(); - this.source = create.source(); - this.sourceOffset = create.sourceOffset(); - this.sourceLength = create.sourceLength(); + this.source = new BytesHolder(create.source(), create.sourceOffset(), create.sourceLength()); this.routing = create.routing(); this.parent = create.parent(); this.timestamp = create.timestamp(); @@ -269,9 +264,7 @@ public interface Translog extends IndexShardComponent { public Create(String type, String id, byte[] source) { this.id = id; this.type = type; - this.source = source; - this.sourceOffset = 0; - this.sourceLength = source.length; + this.source = new BytesHolder(source); } @Override @@ -281,25 +274,17 @@ public interface Translog extends IndexShardComponent { @Override public long estimateSize() { - return ((id.length() + type.length()) * 2) + source.length + 12; + return ((id.length() + type.length()) * 2) + source.length() + 12; } public String id() { return this.id; } - public byte[] source() { + public BytesHolder source() { return this.source; } - public int sourceOffset() { - return this.sourceOffset; - } - - public int sourceLength() { - return this.sourceLength; - } - public String type() { return this.type; } @@ -325,34 +310,8 @@ public interface Translog extends IndexShardComponent { } @Override - public Source readSource(BytesStreamInput in) throws IOException { - int version = in.readVInt(); // version - id = in.readUTF(); - type = in.readUTF(); - - int length = in.readVInt(); - int offset = in.position(); - BytesHolder source = new BytesHolder(in.underlyingBuffer(), offset, length); - in.skip(length); - if (version >= 1) { - if (in.readBoolean()) { - routing = in.readUTF(); - } - } - if (version >= 2) { - if (in.readBoolean()) { - parent = in.readUTF(); - } - } - if (version >= 3) { - this.version = in.readLong(); - } - if (version >= 4) { - this.timestamp = in.readLong(); - } - if (version >= 5) { - this.ttl = in.readLong(); - } + public Source readSource(StreamInput in) throws IOException { + readFrom(in); return new Source(source, routing, parent, timestamp, ttl); } @@ -361,10 +320,7 @@ public interface Translog extends IndexShardComponent { int version = in.readVInt(); // version id = in.readUTF(); type = in.readUTF(); - sourceOffset = 0; - sourceLength = in.readVInt(); - source = new byte[sourceLength]; - in.readFully(source); + source = in.readBytesReference(); if (version >= 1) { if (in.readBoolean()) { routing = in.readUTF(); @@ -391,8 +347,7 @@ public interface Translog extends IndexShardComponent { out.writeVInt(5); // version out.writeUTF(id); out.writeUTF(type); - out.writeVInt(sourceLength); - out.writeBytes(source, sourceOffset, sourceLength); + out.writeBytesHolder(source); if (routing == null) { out.writeBoolean(false); } else { @@ -415,9 +370,7 @@ public interface Translog extends IndexShardComponent { private String id; private String type; private long version; - private byte[] source; - private int sourceOffset; - private int sourceLength; + private BytesHolder source; private String routing; private String parent; private long timestamp; @@ -429,9 +382,7 @@ public interface Translog extends IndexShardComponent { public Index(Engine.Index index) { this.id = index.id(); this.type = index.type(); - this.source = index.source(); - this.sourceOffset = index.sourceOffset(); - this.sourceLength = index.sourceLength(); + this.source = new BytesHolder(index.source(), index.sourceOffset(), index.sourceLength()); this.routing = index.routing(); this.parent = index.parent(); this.version = index.version(); @@ -442,9 +393,7 @@ public interface Translog extends IndexShardComponent { public Index(String type, String id, byte[] source) { this.type = type; this.id = id; - this.source = source; - this.sourceOffset = 0; - this.sourceLength = source.length; + this.source = new BytesHolder(source); } @Override @@ -454,7 +403,7 @@ public interface Translog extends IndexShardComponent { @Override public long estimateSize() { - return ((id.length() + type.length()) * 2) + source.length + 12; + return ((id.length() + type.length()) * 2) + source.length() + 12; } public String type() { @@ -481,51 +430,17 @@ public interface Translog extends IndexShardComponent { return this.ttl; } - public byte[] source() { + public BytesHolder source() { return this.source; } - public int sourceOffset() { - return this.sourceOffset; - } - - public int sourceLength() { - return this.sourceLength; - } - public long version() { return this.version; } @Override - public Source readSource(BytesStreamInput in) throws IOException { - int version = in.readVInt(); // version - id = in.readUTF(); - type = in.readUTF(); - - int length = in.readVInt(); - int offset = in.position(); - BytesHolder source = new BytesHolder(in.underlyingBuffer(), offset, length); - in.skip(length); - if (version >= 1) { - if (in.readBoolean()) { - routing = in.readUTF(); - } - } - if (version >= 2) { - if (in.readBoolean()) { - parent = in.readUTF(); - } - } - if (version >= 3) { - this.version = in.readLong(); - } - if (version >= 4) { - this.timestamp = in.readLong(); - } - if (version >= 5) { - this.ttl = in.readLong(); - } + public Source readSource(StreamInput in) throws IOException { + readFrom(in); return new Source(source, routing, parent, timestamp, ttl); } @@ -534,10 +449,7 @@ public interface Translog extends IndexShardComponent { int version = in.readVInt(); // version id = in.readUTF(); type = in.readUTF(); - sourceOffset = 0; - sourceLength = in.readVInt(); - source = new byte[sourceLength]; - in.readFully(source); + source = in.readBytesReference(); if (version >= 1) { if (in.readBoolean()) { routing = in.readUTF(); @@ -564,8 +476,7 @@ public interface Translog extends IndexShardComponent { out.writeVInt(5); // version out.writeUTF(id); out.writeUTF(type); - out.writeVInt(sourceLength); - out.writeBytes(source, sourceOffset, sourceLength); + out.writeBytesHolder(source); if (routing == null) { out.writeBoolean(false); } else { @@ -619,7 +530,7 @@ public interface Translog extends IndexShardComponent { } @Override - public Source readSource(BytesStreamInput in) throws IOException { + public Source readSource(StreamInput in) throws IOException { throw new ElasticSearchIllegalStateException("trying to read doc source from delete operation"); } @@ -683,7 +594,7 @@ public interface Translog extends IndexShardComponent { } @Override - public Source readSource(BytesStreamInput in) throws IOException { + public Source readSource(StreamInput in) throws IOException { throw new ElasticSearchIllegalStateException("trying to read doc source from delete_by_query operation"); } diff --git a/src/main/java/org/elasticsearch/index/translog/TranslogStreams.java b/src/main/java/org/elasticsearch/index/translog/TranslogStreams.java index b2aa1ae6784..230575d2a48 100644 --- a/src/main/java/org/elasticsearch/index/translog/TranslogStreams.java +++ b/src/main/java/org/elasticsearch/index/translog/TranslogStreams.java @@ -54,7 +54,7 @@ public class TranslogStreams { } public static Translog.Source readSource(byte[] data) throws IOException { - BytesStreamInput in = new BytesStreamInput(data); + BytesStreamInput in = new BytesStreamInput(data, false); in.readInt(); // the size header Translog.Operation.Type type = Translog.Operation.Type.fromId(in.readByte()); Translog.Operation operation; diff --git a/src/main/java/org/elasticsearch/index/translog/fs/FsChannelSnapshot.java b/src/main/java/org/elasticsearch/index/translog/fs/FsChannelSnapshot.java index d3312f3700c..8b545945259 100644 --- a/src/main/java/org/elasticsearch/index/translog/fs/FsChannelSnapshot.java +++ b/src/main/java/org/elasticsearch/index/translog/fs/FsChannelSnapshot.java @@ -120,7 +120,7 @@ public class FsChannelSnapshot implements Translog.Snapshot { channel.read(cacheBuffer, position); cacheBuffer.flip(); position += opSize; - lastOperationRead = TranslogStreams.readTranslogOperation(new BytesStreamInput(cacheBuffer.array(), 0, opSize)); + lastOperationRead = TranslogStreams.readTranslogOperation(new BytesStreamInput(cacheBuffer.array(), 0, opSize, true)); return true; } catch (Exception e) { return false; diff --git a/src/main/java/org/elasticsearch/rest/action/support/RestXContentBuilder.java b/src/main/java/org/elasticsearch/rest/action/support/RestXContentBuilder.java index ed262178ecd..37f60404e7a 100644 --- a/src/main/java/org/elasticsearch/rest/action/support/RestXContentBuilder.java +++ b/src/main/java/org/elasticsearch/rest/action/support/RestXContentBuilder.java @@ -68,7 +68,7 @@ public class RestXContentBuilder { public static void restDocumentSource(byte[] source, int offset, int length, XContentBuilder builder, ToXContent.Params params) throws IOException { if (LZF.isCompressed(source, offset, length)) { - BytesStreamInput siBytes = new BytesStreamInput(source, offset, length); + BytesStreamInput siBytes = new BytesStreamInput(source, offset, length, false); LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes); XContentType contentType = XContentFactory.xContentType(siLzf); siLzf.resetToBufferStart(); diff --git a/src/main/java/org/elasticsearch/transport/local/LocalTransport.java b/src/main/java/org/elasticsearch/transport/local/LocalTransport.java index 08a6daee3f2..325532164cf 100644 --- a/src/main/java/org/elasticsearch/transport/local/LocalTransport.java +++ b/src/main/java/org/elasticsearch/transport/local/LocalTransport.java @@ -195,7 +195,7 @@ public class LocalTransport extends AbstractLifecycleComponent implem void messageReceived(byte[] data, String action, LocalTransport sourceTransport, @Nullable final Long sendRequestId) { transportServiceAdapter.received(data.length); - StreamInput stream = new BytesStreamInput(data); + StreamInput stream = new BytesStreamInput(data, false); stream = CachedStreamInput.cachedHandles(stream); try { diff --git a/src/test/java/org/elasticsearch/test/unit/cluster/serialization/ClusterSerializationTests.java b/src/test/java/org/elasticsearch/test/unit/cluster/serialization/ClusterSerializationTests.java index 8d8fda75987..a05207f418d 100644 --- a/src/test/java/org/elasticsearch/test/unit/cluster/serialization/ClusterSerializationTests.java +++ b/src/test/java/org/elasticsearch/test/unit/cluster/serialization/ClusterSerializationTests.java @@ -85,7 +85,7 @@ public class ClusterSerializationTests { BytesStreamOutput outStream = new BytesStreamOutput(); RoutingTable.Builder.writeTo(source, outStream); - BytesStreamInput inStream = new BytesStreamInput(outStream.copiedByteArray()); + BytesStreamInput inStream = new BytesStreamInput(outStream.copiedByteArray(), false); RoutingTable target = RoutingTable.Builder.readFrom(inStream); assertThat(target.prettyPrint(), equalTo(source.prettyPrint())); diff --git a/src/test/java/org/elasticsearch/test/unit/common/io/streams/BytesStreamsTests.java b/src/test/java/org/elasticsearch/test/unit/common/io/streams/BytesStreamsTests.java index 9222b328750..36419eb67e8 100644 --- a/src/test/java/org/elasticsearch/test/unit/common/io/streams/BytesStreamsTests.java +++ b/src/test/java/org/elasticsearch/test/unit/common/io/streams/BytesStreamsTests.java @@ -49,7 +49,7 @@ public class BytesStreamsTests { out.writeUTF("hello"); out.writeUTF("goodbye"); - BytesStreamInput in = new BytesStreamInput(out.copiedByteArray()); + BytesStreamInput in = new BytesStreamInput(out.copiedByteArray(), false); assertThat(in.readBoolean(), equalTo(false)); assertThat(in.readByte(), equalTo((byte) 1)); assertThat(in.readShort(), equalTo((short) -1)); diff --git a/src/test/java/org/elasticsearch/test/unit/common/io/streams/HandlesStreamsTests.java b/src/test/java/org/elasticsearch/test/unit/common/io/streams/HandlesStreamsTests.java index f7b33789936..56b2a438f86 100644 --- a/src/test/java/org/elasticsearch/test/unit/common/io/streams/HandlesStreamsTests.java +++ b/src/test/java/org/elasticsearch/test/unit/common/io/streams/HandlesStreamsTests.java @@ -47,7 +47,7 @@ public class HandlesStreamsTests { out.writeUTF(higherThresholdValue); out.writeUTF(lowerThresholdValue); - HandlesStreamInput in = new HandlesStreamInput(new BytesStreamInput(bytesOut.copiedByteArray())); + HandlesStreamInput in = new HandlesStreamInput(new BytesStreamInput(bytesOut.copiedByteArray(), false)); assertThat(in.readUTF(), equalTo(lowerThresholdValue)); assertThat(in.readUTF(), equalTo(higherThresholdValue)); assertThat(in.readInt(), equalTo(1)); diff --git a/src/test/java/org/elasticsearch/test/unit/index/engine/AbstractSimpleEngineTests.java b/src/test/java/org/elasticsearch/test/unit/index/engine/AbstractSimpleEngineTests.java index 3b6705248af..7d22fe2c8c4 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/engine/AbstractSimpleEngineTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/engine/AbstractSimpleEngineTests.java @@ -396,7 +396,7 @@ public abstract class AbstractSimpleEngineTests { MatcherAssert.assertThat(snapshotIndexCommit1, SnapshotIndexCommitExistsMatcher.snapshotIndexCommitExists()); assertThat(translogSnapshot1.hasNext(), equalTo(true)); Translog.Create create1 = (Translog.Create) translogSnapshot1.next(); - assertThat(create1.source(), equalTo(B_1)); + assertThat(create1.source().copyBytes(), equalTo(B_1)); assertThat(translogSnapshot1.hasNext(), equalTo(false)); Future future = executorService.submit(new Callable() { @@ -429,7 +429,7 @@ public abstract class AbstractSimpleEngineTests { assertThat(snapshotIndexCommit2.getSegmentsFileName(), not(equalTo(snapshotIndexCommit1.getSegmentsFileName()))); assertThat(translogSnapshot2.hasNext(), equalTo(true)); Translog.Create create3 = (Translog.Create) translogSnapshot2.next(); - assertThat(create3.source(), equalTo(B_3)); + assertThat(create3.source().copyBytes(), equalTo(B_3)); assertThat(translogSnapshot2.hasNext(), equalTo(false)); return null; } @@ -503,7 +503,7 @@ public abstract class AbstractSimpleEngineTests { public void phase2(Translog.Snapshot snapshot) throws EngineException { assertThat(snapshot.hasNext(), equalTo(true)); Translog.Create create = (Translog.Create) snapshot.next(); - assertThat(create.source(), equalTo(B_2)); + assertThat(create.source().copyBytes(), equalTo(B_2)); assertThat(snapshot.hasNext(), equalTo(false)); } @@ -535,7 +535,7 @@ public abstract class AbstractSimpleEngineTests { assertThat(snapshot.hasNext(), equalTo(true)); Translog.Create create = (Translog.Create) snapshot.next(); assertThat(snapshot.hasNext(), equalTo(false)); - assertThat(create.source(), equalTo(B_2)); + assertThat(create.source().copyBytes(), equalTo(B_2)); // add for phase3 ParsedDocument doc3 = new ParsedDocument("3", "3", "test", null, -1, -1, doc().add(uidField("3")).add(field("value", "test")).build(), Lucene.STANDARD_ANALYZER, B_3, false); @@ -547,7 +547,7 @@ public abstract class AbstractSimpleEngineTests { assertThat(snapshot.hasNext(), equalTo(true)); Translog.Create create = (Translog.Create) snapshot.next(); assertThat(snapshot.hasNext(), equalTo(false)); - assertThat(create.source(), equalTo(B_3)); + assertThat(create.source().copyBytes(), equalTo(B_3)); } }); diff --git a/src/test/java/org/elasticsearch/test/unit/index/translog/AbstractSimpleTranslogTests.java b/src/test/java/org/elasticsearch/test/unit/index/translog/AbstractSimpleTranslogTests.java index e233193ca82..5bc8607b5b9 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/translog/AbstractSimpleTranslogTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/translog/AbstractSimpleTranslogTests.java @@ -140,11 +140,11 @@ public abstract class AbstractSimpleTranslogTests { assertThat(snapshot.hasNext(), equalTo(true)); Translog.Create create = (Translog.Create) snapshot.next(); - assertThat(create.source(), equalTo(new byte[]{1})); + assertThat(create.source().copyBytes(), equalTo(new byte[]{1})); assertThat(snapshot.hasNext(), equalTo(true)); Translog.Index index = (Translog.Index) snapshot.next(); - assertThat(index.source(), equalTo(new byte[]{2})); + assertThat(index.source().copyBytes(), equalTo(new byte[]{2})); assertThat(snapshot.hasNext(), equalTo(true)); Translog.Delete delete = (Translog.Delete) snapshot.next(); @@ -183,7 +183,7 @@ public abstract class AbstractSimpleTranslogTests { snapshot = translog.snapshot(); assertThat(snapshot.hasNext(), equalTo(true)); Translog.Create create = (Translog.Create) snapshot.next(); - assertThat(create.source(), equalTo(new byte[]{1})); + assertThat(create.source().copyBytes(), equalTo(new byte[]{1})); snapshot.release(); Translog.Snapshot snapshot1 = translog.snapshot(); @@ -201,7 +201,7 @@ public abstract class AbstractSimpleTranslogTests { snapshot = translog.snapshot(snapshot1); assertThat(snapshot.hasNext(), equalTo(true)); Translog.Index index = (Translog.Index) snapshot.next(); - assertThat(index.source(), equalTo(new byte[]{2})); + assertThat(index.source().copyBytes(), equalTo(new byte[]{2})); assertThat(snapshot.hasNext(), equalTo(false)); assertThat(snapshot.estimatedTotalOperations(), equalTo(2)); snapshot.release(); @@ -230,7 +230,7 @@ public abstract class AbstractSimpleTranslogTests { snapshot = translog.snapshot(actualSnapshot); assertThat(snapshot.hasNext(), equalTo(true)); Translog.Index index = (Translog.Index) snapshot.next(); - assertThat(index.source(), equalTo(new byte[]{3})); + assertThat(index.source().copyBytes(), equalTo(new byte[]{3})); assertThat(snapshot.hasNext(), equalTo(false)); actualSnapshot.release(); From 0cc906aa212989db667692a1ea0fb5291a00fed5 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 20:07:13 +0200 Subject: [PATCH 141/270] read reference binary data in publish cluster state --- .../zen/publish/PublishClusterStateAction.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java index 51970a98886..44e6ab565ff 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java +++ b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java @@ -21,6 +21,7 @@ package org.elasticsearch.discovery.zen.publish; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.io.stream.*; import org.elasticsearch.common.settings.Settings; @@ -97,25 +98,23 @@ public class PublishClusterStateAction extends AbstractComponent { class PublishClusterStateRequest implements Streamable { - private byte[] clusterStateInBytes; + BytesHolder clusterStateInBytes; private PublishClusterStateRequest() { } private PublishClusterStateRequest(byte[] clusterStateInBytes) { - this.clusterStateInBytes = clusterStateInBytes; + this.clusterStateInBytes = new BytesHolder(clusterStateInBytes); } @Override public void readFrom(StreamInput in) throws IOException { - clusterStateInBytes = new byte[in.readVInt()]; - in.readFully(clusterStateInBytes); + clusterStateInBytes = in.readBytesReference(); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(clusterStateInBytes.length); - out.writeBytes(clusterStateInBytes); + out.writeBytesHolder(clusterStateInBytes); } } @@ -130,7 +129,7 @@ public class PublishClusterStateAction extends AbstractComponent { @Override public void messageReceived(PublishClusterStateRequest request, TransportChannel channel) throws Exception { - StreamInput in = CachedStreamInput.cachedHandlesLzf(new BytesStreamInput(request.clusterStateInBytes, false)); + StreamInput in = CachedStreamInput.cachedHandlesLzf(new BytesStreamInput(request.clusterStateInBytes.bytes(), request.clusterStateInBytes.offset(), request.clusterStateInBytes.length(), false)); ClusterState clusterState = ClusterState.Builder.readFrom(in, nodesProvider.nodes().localNode()); listener.onNewClusterState(clusterState); channel.sendResponse(VoidStreamable.INSTANCE); From 0f1b3f0457c3e59fcbf1d535cf8bc15ef0e4e561 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 20:52:48 +0200 Subject: [PATCH 142/270] delete by query to use byte reference serialization --- .../deletebyquery/DeleteByQueryRequest.java | 23 +++++++--------- .../IndexDeleteByQueryRequest.java | 26 ++++--------------- .../ShardDeleteByQueryRequest.java | 15 +++++------ .../elasticsearch/index/engine/Engine.java | 7 ++--- .../index/shard/service/IndexShard.java | 3 ++- .../shard/service/InternalIndexShard.java | 5 ++-- .../index/translog/Translog.java | 14 +++++----- .../translog/AbstractSimpleTranslogTests.java | 4 +-- 8 files changed, 37 insertions(+), 60 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java b/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java index 7ecf5ed8b47..8b135edb086 100644 --- a/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java +++ b/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java @@ -26,10 +26,7 @@ import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.support.replication.IndicesReplicationOperationRequest; import org.elasticsearch.action.support.replication.ReplicationType; import org.elasticsearch.client.Requests; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Required; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.Unicode; +import org.elasticsearch.common.*; import org.elasticsearch.common.io.BytesStream; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -52,7 +49,6 @@ import static org.elasticsearch.action.Actions.addValidationError; *

    The request requires the query source to be set either using {@link #query(org.elasticsearch.index.query.QueryBuilder)}, * or {@link #query(byte[])}. * - * * @see DeleteByQueryResponse * @see org.elasticsearch.client.Requests#deleteByQueryRequest(String...) * @see org.elasticsearch.client.Client#deleteByQuery(DeleteByQueryRequest) @@ -110,13 +106,13 @@ public class DeleteByQueryRequest extends IndicesReplicationOperationRequest { /** * The query source to execute. */ - byte[] querySource() { - if (querySourceUnsafe || querySourceOffset > 0) { + BytesHolder querySource() { + if (querySourceUnsafe) { querySource = Arrays.copyOfRange(querySource, querySourceOffset, querySourceOffset + querySourceLength); querySourceOffset = 0; querySourceUnsafe = false; } - return querySource; + return new BytesHolder(querySource, querySourceOffset, querySourceLength); } /** @@ -274,11 +270,11 @@ public class DeleteByQueryRequest extends IndicesReplicationOperationRequest { public void readFrom(StreamInput in) throws IOException { super.readFrom(in); + BytesHolder bytes = in.readBytesReference(); querySourceUnsafe = false; - querySourceOffset = 0; - querySourceLength = in.readVInt(); - querySource = new byte[querySourceLength]; - in.readFully(querySource); + querySource = bytes.bytes(); + querySourceOffset = bytes.offset(); + querySourceLength = bytes.length(); if (in.readBoolean()) { routing = in.readUTF(); @@ -298,8 +294,7 @@ public class DeleteByQueryRequest extends IndicesReplicationOperationRequest { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeVInt(querySourceLength); - out.writeBytes(querySource, querySourceOffset, querySourceLength); + out.writeBytesHolder(querySource, querySourceOffset, querySourceLength); if (routing == null) { out.writeBoolean(false); diff --git a/src/main/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryRequest.java b/src/main/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryRequest.java index 48134cc2478..fd86bbb52c8 100644 --- a/src/main/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryRequest.java +++ b/src/main/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryRequest.java @@ -22,13 +22,12 @@ package org.elasticsearch.action.deletebyquery; import gnu.trove.set.hash.THashSet; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.replication.IndexReplicationOperationRequest; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Required; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.QueryBuilder; import java.io.IOException; import java.util.Set; @@ -37,12 +36,10 @@ import static org.elasticsearch.action.Actions.addValidationError; /** * Delete by query request to execute on a specific index. - * - * */ public class IndexDeleteByQueryRequest extends IndexReplicationOperationRequest { - private byte[] querySource; + private BytesHolder querySource; private String[] types = Strings.EMPTY_ARRAY; @Nullable private Set routing; @@ -63,7 +60,7 @@ public class IndexDeleteByQueryRequest extends IndexReplicationOperationRequest IndexDeleteByQueryRequest() { } - byte[] querySource() { + BytesHolder querySource() { return querySource; } @@ -76,17 +73,6 @@ public class IndexDeleteByQueryRequest extends IndexReplicationOperationRequest return validationException; } - @Required - public IndexDeleteByQueryRequest querySource(QueryBuilder queryBuilder) { - return querySource(queryBuilder.buildAsBytes()); - } - - @Required - public IndexDeleteByQueryRequest querySource(byte[] querySource) { - this.querySource = querySource; - return this; - } - Set routing() { return this.routing; } @@ -106,8 +92,7 @@ public class IndexDeleteByQueryRequest extends IndexReplicationOperationRequest public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - querySource = new byte[in.readVInt()]; - in.readFully(querySource); + querySource = in.readBytesReference(); int typesSize = in.readVInt(); if (typesSize > 0) { types = new String[typesSize]; @@ -133,8 +118,7 @@ public class IndexDeleteByQueryRequest extends IndexReplicationOperationRequest public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeVInt(querySource.length); - out.writeBytes(querySource); + out.writeBytesHolder(querySource); out.writeVInt(types.length); for (String type : types) { out.writeUTF(type); diff --git a/src/main/java/org/elasticsearch/action/deletebyquery/ShardDeleteByQueryRequest.java b/src/main/java/org/elasticsearch/action/deletebyquery/ShardDeleteByQueryRequest.java index f9f5baebf43..c1a68772d01 100644 --- a/src/main/java/org/elasticsearch/action/deletebyquery/ShardDeleteByQueryRequest.java +++ b/src/main/java/org/elasticsearch/action/deletebyquery/ShardDeleteByQueryRequest.java @@ -22,6 +22,7 @@ package org.elasticsearch.action.deletebyquery; import gnu.trove.set.hash.THashSet; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.replication.ShardReplicationOperationRequest; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.Unicode; @@ -36,13 +37,11 @@ import static org.elasticsearch.action.Actions.addValidationError; /** * Delete by query request to execute on a specific shard. - * - * */ public class ShardDeleteByQueryRequest extends ShardReplicationOperationRequest { private int shardId; - private byte[] querySource; + private BytesHolder querySource; private String[] types = Strings.EMPTY_ARRAY; @Nullable private Set routing; @@ -77,7 +76,7 @@ public class ShardDeleteByQueryRequest extends ShardReplicationOperationRequest return this.shardId; } - public byte[] querySource() { + BytesHolder querySource() { return querySource; } @@ -96,8 +95,7 @@ public class ShardDeleteByQueryRequest extends ShardReplicationOperationRequest @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - querySource = new byte[in.readVInt()]; - in.readFully(querySource); + querySource = in.readBytesReference(); shardId = in.readVInt(); int typesSize = in.readVInt(); if (typesSize > 0) { @@ -125,8 +123,7 @@ public class ShardDeleteByQueryRequest extends ShardReplicationOperationRequest @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeVInt(querySource.length); - out.writeBytes(querySource); + out.writeBytesHolder(querySource); out.writeVInt(shardId); out.writeVInt(types.length); for (String type : types) { @@ -154,7 +151,7 @@ public class ShardDeleteByQueryRequest extends ShardReplicationOperationRequest public String toString() { String sSource = "_na_"; try { - sSource = Unicode.fromBytes(querySource); + sSource = Unicode.fromBytes(querySource.bytes(), querySource.offset(), querySource.length()); } catch (Exception e) { // ignore } diff --git a/src/main/java/org/elasticsearch/index/engine/Engine.java b/src/main/java/org/elasticsearch/index/engine/Engine.java index 37f358cf7a9..78d0159f0b1 100644 --- a/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -27,6 +27,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.Filter; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.CloseableComponent; import org.elasticsearch.common.lease.Releasable; @@ -692,7 +693,7 @@ public interface Engine extends IndexShardComponent, CloseableComponent { static class DeleteByQuery { private final Query query; - private final byte[] source; + private final BytesHolder source; private final String[] filteringAliases; private final Filter aliasFilter; private final String[] types; @@ -700,7 +701,7 @@ public interface Engine extends IndexShardComponent, CloseableComponent { private long startTime; private long endTime; - public DeleteByQuery(Query query, byte[] source, @Nullable String[] filteringAliases, @Nullable Filter aliasFilter, String... types) { + public DeleteByQuery(Query query, BytesHolder source, @Nullable String[] filteringAliases, @Nullable Filter aliasFilter, String... types) { this.query = query; this.source = source; this.types = types; @@ -712,7 +713,7 @@ public interface Engine extends IndexShardComponent, CloseableComponent { return this.query; } - public byte[] source() { + public BytesHolder source() { return this.source; } diff --git a/src/main/java/org/elasticsearch/index/shard/service/IndexShard.java b/src/main/java/org/elasticsearch/index/shard/service/IndexShard.java index d39c6eb9fd1..2139aa1da81 100644 --- a/src/main/java/org/elasticsearch/index/shard/service/IndexShard.java +++ b/src/main/java/org/elasticsearch/index/shard/service/IndexShard.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.shard.service; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.EngineException; @@ -83,7 +84,7 @@ public interface IndexShard extends IndexShardComponent { void delete(Engine.Delete delete) throws ElasticSearchException; - Engine.DeleteByQuery prepareDeleteByQuery(byte[] querySource, @Nullable String[] filteringAliases, String... types) throws ElasticSearchException; + Engine.DeleteByQuery prepareDeleteByQuery(BytesHolder querySource, @Nullable String[] filteringAliases, String... types) throws ElasticSearchException; void deleteByQuery(Engine.DeleteByQuery deleteByQuery) throws ElasticSearchException; diff --git a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java index 24e70962431..c059e092d2b 100644 --- a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java +++ b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java @@ -30,6 +30,7 @@ import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.ElasticSearchIllegalStateException; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; @@ -345,12 +346,12 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I } @Override - public Engine.DeleteByQuery prepareDeleteByQuery(byte[] querySource, @Nullable String[] filteringAliases, String... types) throws ElasticSearchException { + public Engine.DeleteByQuery prepareDeleteByQuery(BytesHolder querySource, @Nullable String[] filteringAliases, String... types) throws ElasticSearchException { long startTime = System.nanoTime(); if (types == null) { types = Strings.EMPTY_ARRAY; } - Query query = queryParserService.parse(querySource).query(); + Query query = queryParserService.parse(querySource.bytes(), querySource.offset(), querySource.length()).query(); query = filterQueryIfNeeded(query, types); Filter aliasFilter = indexAliasesService.aliasFilter(filteringAliases); diff --git a/src/main/java/org/elasticsearch/index/translog/Translog.java b/src/main/java/org/elasticsearch/index/translog/Translog.java index 0907695ec89..60b6850036e 100644 --- a/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -553,7 +553,7 @@ public interface Translog extends IndexShardComponent { } static class DeleteByQuery implements Operation { - private byte[] source; + private BytesHolder source; @Nullable private String[] filteringAliases; private String[] types = Strings.EMPTY_ARRAY; @@ -565,7 +565,7 @@ public interface Translog extends IndexShardComponent { this(deleteByQuery.source(), deleteByQuery.filteringAliases(), deleteByQuery.types()); } - public DeleteByQuery(byte[] source, String[] filteringAliases, String... types) { + public DeleteByQuery(BytesHolder source, String[] filteringAliases, String... types) { this.source = source; this.types = types == null ? Strings.EMPTY_ARRAY : types; this.filteringAliases = filteringAliases; @@ -578,10 +578,10 @@ public interface Translog extends IndexShardComponent { @Override public long estimateSize() { - return source.length + 8; + return source.length() + 8; } - public byte[] source() { + public BytesHolder source() { return this.source; } @@ -601,8 +601,7 @@ public interface Translog extends IndexShardComponent { @Override public void readFrom(StreamInput in) throws IOException { int version = in.readVInt(); // version - source = new byte[in.readVInt()]; - in.readFully(source); + source = in.readBytesReference(); if (version < 2) { // for query_parser_name, which was removed if (in.readBoolean()) { @@ -630,8 +629,7 @@ public interface Translog extends IndexShardComponent { @Override public void writeTo(StreamOutput out) throws IOException { out.writeVInt(2); // version - out.writeVInt(source.length); - out.writeBytes(source); + out.writeBytesHolder(source); out.writeVInt(types.length); for (String type : types) { out.writeUTF(type); diff --git a/src/test/java/org/elasticsearch/test/unit/index/translog/AbstractSimpleTranslogTests.java b/src/test/java/org/elasticsearch/test/unit/index/translog/AbstractSimpleTranslogTests.java index 5bc8607b5b9..eb6a8b1ca27 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/translog/AbstractSimpleTranslogTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/translog/AbstractSimpleTranslogTests.java @@ -130,7 +130,7 @@ public abstract class AbstractSimpleTranslogTests { assertThat(snapshot.estimatedTotalOperations(), equalTo(3)); snapshot.release(); - translog.add(new Translog.DeleteByQuery(new byte[]{4}, null)); + translog.add(new Translog.DeleteByQuery(new BytesHolder(new byte[]{4}), null)); snapshot = translog.snapshot(); MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(4)); assertThat(snapshot.estimatedTotalOperations(), equalTo(4)); @@ -152,7 +152,7 @@ public abstract class AbstractSimpleTranslogTests { assertThat(snapshot.hasNext(), equalTo(true)); Translog.DeleteByQuery deleteByQuery = (Translog.DeleteByQuery) snapshot.next(); - assertThat(deleteByQuery.source(), equalTo(new byte[]{4})); + assertThat(deleteByQuery.source().copyBytes(), equalTo(new byte[]{4})); assertThat(snapshot.hasNext(), equalTo(false)); From bb1b46431cb0ffbf587b341559741599821c94eb Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 23:51:34 +0200 Subject: [PATCH 143/270] move validate query to read bytes ref --- .../query/ShardValidateQueryRequest.java | 29 +++++-------------- .../query/TransportValidateQueryAction.java | 4 +-- .../validate/query/ValidateQueryRequest.java | 28 ++++++------------ 3 files changed, 18 insertions(+), 43 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java index 6978bbbb1f8..e725b1dd98d 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.indices.validate.query; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationRequest; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -29,14 +30,10 @@ import java.io.IOException; /** * Internal validate request executed directly against a specific index shard. - * - * */ class ShardValidateQueryRequest extends BroadcastShardOperationRequest { - private byte[] querySource; - private int querySourceOffset; - private int querySourceLength; + private BytesHolder querySource; private String[] types = Strings.EMPTY_ARRAY; @@ -50,24 +47,14 @@ class ShardValidateQueryRequest extends BroadcastShardOperationRequest { public ShardValidateQueryRequest(String index, int shardId, @Nullable String[] filteringAliases, ValidateQueryRequest request) { super(index, shardId); this.querySource = request.querySource(); - this.querySourceOffset = request.querySourceOffset(); - this.querySourceLength = request.querySourceLength(); this.types = request.types(); this.filteringAliases = filteringAliases; } - public byte[] querySource() { + public BytesHolder querySource() { return querySource; } - public int querySourceOffset() { - return querySourceOffset; - } - - public int querySourceLength() { - return querySourceLength; - } - public String[] types() { return this.types; } @@ -79,10 +66,8 @@ class ShardValidateQueryRequest extends BroadcastShardOperationRequest { @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - querySourceLength = in.readVInt(); - querySourceOffset = 0; - querySource = new byte[querySourceLength]; - in.readFully(querySource); + querySource = in.readBytesReference(); + int typesSize = in.readVInt(); if (typesSize > 0) { types = new String[typesSize]; @@ -102,8 +87,8 @@ class ShardValidateQueryRequest extends BroadcastShardOperationRequest { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeVInt(querySourceLength); - out.writeBytes(querySource, querySourceOffset, querySourceLength); + out.writeBytesHolder(querySource); + out.writeVInt(types.length); for (String type : types) { out.writeUTF(type); diff --git a/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java index 84cd7156c50..a8fec5bce64 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java @@ -136,11 +136,11 @@ public class TransportValidateQueryAction extends TransportBroadcastOperationAct protected ShardValidateQueryResponse shardOperation(ShardValidateQueryRequest request) throws ElasticSearchException { IndexQueryParserService queryParserService = indicesService.indexServiceSafe(request.index()).queryParserService(); boolean valid; - if (request.querySourceLength() == 0) { + if (request.querySource().length() == 0) { valid = true; } else { try { - queryParserService.parse(request.querySource(), request.querySourceOffset(), request.querySourceLength()); + queryParserService.parse(request.querySource().bytes(), request.querySource().offset(), request.querySource().length()); valid = true; } catch (QueryParsingException e) { valid = false; diff --git a/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java index f5264ee7e34..18030e6d175 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.broadcast.BroadcastOperationRequest; import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; import org.elasticsearch.client.Requests; +import org.elasticsearch.common.BytesHolder; import org.elasticsearch.common.Required; import org.elasticsearch.common.Strings; import org.elasticsearch.common.Unicode; @@ -45,8 +46,6 @@ import java.util.Map; *

    *

    The request requires the query source to be set either using {@link #query(org.elasticsearch.index.query.QueryBuilder)}, * or {@link #query(byte[])}. - * - * */ public class ValidateQueryRequest extends BroadcastOperationRequest { @@ -111,16 +110,8 @@ public class ValidateQueryRequest extends BroadcastOperationRequest { /** * The query source to execute. */ - byte[] querySource() { - return querySource; - } - - int querySourceOffset() { - return querySourceOffset; - } - - int querySourceLength() { - return querySourceLength; + BytesHolder querySource() { + return new BytesHolder(querySource, querySourceOffset, querySourceLength); } /** @@ -218,11 +209,11 @@ public class ValidateQueryRequest extends BroadcastOperationRequest { public void readFrom(StreamInput in) throws IOException { super.readFrom(in); + BytesHolder bytes = in.readBytesReference(); querySourceUnsafe = false; - querySourceOffset = 0; - querySourceLength = in.readVInt(); - querySource = new byte[querySourceLength]; - in.readFully(querySource); + querySource = bytes.bytes(); + querySourceOffset = bytes.offset(); + querySourceLength = bytes.length(); int typesSize = in.readVInt(); if (typesSize > 0) { @@ -237,8 +228,7 @@ public class ValidateQueryRequest extends BroadcastOperationRequest { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeVInt(querySourceLength); - out.writeBytes(querySource, querySourceOffset, querySourceLength); + out.writeBytesHolder(querySource, querySourceOffset, querySourceLength); out.writeVInt(types.length); for (String type : types) { @@ -248,6 +238,6 @@ public class ValidateQueryRequest extends BroadcastOperationRequest { @Override public String toString() { - return "[" + Arrays.toString(indices) + "]" + Arrays.toString(types) + ", querySource[" + Unicode.fromBytes(querySource) + "]"; + return "[" + Arrays.toString(indices) + "]" + Arrays.toString(types) + ", querySource[" + Unicode.fromBytes(querySource, querySourceOffset, querySourceLength) + "]"; } } From 39e41ab819e51eae9db8439238ca996e7728f578 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 8 Jan 2012 23:52:19 +0200 Subject: [PATCH 144/270] no need to raise a node disconnected if the transport service is stopped --- .../java/org/elasticsearch/transport/TransportService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/elasticsearch/transport/TransportService.java b/src/main/java/org/elasticsearch/transport/TransportService.java index 9c00052d356..4f148c7088b 100644 --- a/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/src/main/java/org/elasticsearch/transport/TransportService.java @@ -287,6 +287,9 @@ public class TransportService extends AbstractLifecycleComponent Date: Mon, 9 Jan 2012 00:17:53 +0200 Subject: [PATCH 145/270] query builder builds a "safe" byte array --- .../validate/query/ValidateQueryRequest.java | 4 ++-- .../action/count/CountRequest.java | 4 ++-- .../deletebyquery/DeleteByQueryRequest.java | 4 ++-- .../index/query/BaseQueryBuilder.java | 22 +++---------------- .../index/query/IndexQueryParserService.java | 4 ++-- .../index/query/QueryBuilder.java | 8 ++----- .../query/RestValidateQueryAction.java | 7 +++--- .../rest/action/count/RestCountAction.java | 5 +++-- .../RestDeleteByQueryAction.java | 4 +++- .../rest/action/support/RestActions.java | 3 ++- .../query/SimpleIndexQueryParserTests.java | 8 +++---- 11 files changed, 29 insertions(+), 44 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java index 18030e6d175..bf38df36092 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java @@ -121,11 +121,11 @@ public class ValidateQueryRequest extends BroadcastOperationRequest { */ @Required public ValidateQueryRequest query(QueryBuilder queryBuilder) { - BytesStream bos = queryBuilder.buildAsUnsafeBytes(); + BytesStream bos = queryBuilder.buildAsBytes(); this.querySource = bos.underlyingBytes(); this.querySourceOffset = 0; this.querySourceLength = bos.size(); - this.querySourceUnsafe = true; + this.querySourceUnsafe = false; return this; } diff --git a/src/main/java/org/elasticsearch/action/count/CountRequest.java b/src/main/java/org/elasticsearch/action/count/CountRequest.java index bd604fb5ac3..47d67fd6cce 100644 --- a/src/main/java/org/elasticsearch/action/count/CountRequest.java +++ b/src/main/java/org/elasticsearch/action/count/CountRequest.java @@ -169,11 +169,11 @@ public class CountRequest extends BroadcastOperationRequest { */ @Required public CountRequest query(QueryBuilder queryBuilder) { - BytesStream bos = queryBuilder.buildAsUnsafeBytes(); + BytesStream bos = queryBuilder.buildAsBytes(); this.querySource = bos.underlyingBytes(); this.querySourceOffset = 0; this.querySourceLength = bos.size(); - this.querySourceUnsafe = true; + this.querySourceUnsafe = false; return this; } diff --git a/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java b/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java index 8b135edb086..25f4d917214 100644 --- a/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java +++ b/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java @@ -122,11 +122,11 @@ public class DeleteByQueryRequest extends IndicesReplicationOperationRequest { */ @Required public DeleteByQueryRequest query(QueryBuilder queryBuilder) { - BytesStream bos = queryBuilder.buildAsUnsafeBytes(); + BytesStream bos = queryBuilder.buildAsBytes(); this.querySource = bos.underlyingBytes(); this.querySourceOffset = 0; this.querySourceLength = bos.size(); - this.querySourceUnsafe = true; + this.querySourceUnsafe = false; return this; } diff --git a/src/main/java/org/elasticsearch/index/query/BaseQueryBuilder.java b/src/main/java/org/elasticsearch/index/query/BaseQueryBuilder.java index e5bd7512624..b1c54511871 100644 --- a/src/main/java/org/elasticsearch/index/query/BaseQueryBuilder.java +++ b/src/main/java/org/elasticsearch/index/query/BaseQueryBuilder.java @@ -44,32 +44,16 @@ public abstract class BaseQueryBuilder implements QueryBuilder { } @Override - public BytesStream buildAsUnsafeBytes() throws QueryBuilderException { - return buildAsUnsafeBytes(XContentType.JSON); - } - - @Override - public BytesStream buildAsUnsafeBytes(XContentType contentType) throws QueryBuilderException { - try { - XContentBuilder builder = XContentFactory.contentBuilder(contentType); - toXContent(builder, EMPTY_PARAMS); - return builder.underlyingStream(); - } catch (Exception e) { - throw new QueryBuilderException("Failed to build query", e); - } - } - - @Override - public byte[] buildAsBytes() throws QueryBuilderException { + public BytesStream buildAsBytes() throws QueryBuilderException { return buildAsBytes(XContentType.JSON); } @Override - public byte[] buildAsBytes(XContentType contentType) throws QueryBuilderException { + public BytesStream buildAsBytes(XContentType contentType) throws QueryBuilderException { try { XContentBuilder builder = XContentFactory.contentBuilder(contentType); toXContent(builder, EMPTY_PARAMS); - return builder.copiedBytes(); + return builder.underlyingStream(); } catch (Exception e) { throw new QueryBuilderException("Failed to build query", e); } diff --git a/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java b/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java index 1729808ca7e..142a2d805fd 100644 --- a/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java +++ b/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java @@ -159,8 +159,8 @@ public class IndexQueryParserService extends AbstractIndexComponent { public ParsedQuery parse(QueryBuilder queryBuilder) throws ElasticSearchException { XContentParser parser = null; try { - BytesStream unsafeBytes = queryBuilder.buildAsUnsafeBytes(); - parser = XContentFactory.xContent(unsafeBytes.underlyingBytes(), 0, unsafeBytes.size()).createParser(unsafeBytes.underlyingBytes(), 0, unsafeBytes.size()); + BytesStream bytes = queryBuilder.buildAsBytes(); + parser = XContentFactory.xContent(bytes.underlyingBytes(), 0, bytes.size()).createParser(bytes.underlyingBytes(), 0, bytes.size()); return parse(cache.get(), parser); } catch (QueryParsingException e) { throw e; diff --git a/src/main/java/org/elasticsearch/index/query/QueryBuilder.java b/src/main/java/org/elasticsearch/index/query/QueryBuilder.java index c4101345b30..0fd67ba01a5 100644 --- a/src/main/java/org/elasticsearch/index/query/QueryBuilder.java +++ b/src/main/java/org/elasticsearch/index/query/QueryBuilder.java @@ -28,11 +28,7 @@ import org.elasticsearch.common.xcontent.XContentType; */ public interface QueryBuilder extends ToXContent { - BytesStream buildAsUnsafeBytes() throws QueryBuilderException; + BytesStream buildAsBytes() throws QueryBuilderException; - BytesStream buildAsUnsafeBytes(XContentType contentType) throws QueryBuilderException; - - byte[] buildAsBytes() throws QueryBuilderException; - - byte[] buildAsBytes(XContentType contentType) throws QueryBuilderException; + BytesStream buildAsBytes(XContentType contentType) throws QueryBuilderException; } diff --git a/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java b/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java index 9da168c2c4c..f41225c2b5a 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java @@ -20,11 +20,12 @@ package org.elasticsearch.rest.action.admin.indices.validate.query; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; +import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; import org.elasticsearch.client.Client; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.BytesStream; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.*; @@ -75,9 +76,9 @@ public class RestValidateQueryAction extends BaseRestHandler { if (source != null) { validateQueryRequest.query(source); } else { - byte[] querySource = RestActions.parseQuerySource(request); + BytesStream querySource = RestActions.parseQuerySource(request); if (querySource != null) { - validateQueryRequest.query(querySource); + validateQueryRequest.query(querySource.underlyingBytes(), 0, querySource.size(), false); } } } diff --git a/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java b/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java index d755fcc250c..4cc3a128691 100644 --- a/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java +++ b/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; import org.elasticsearch.client.Client; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.BytesStream; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.*; @@ -76,9 +77,9 @@ public class RestCountAction extends BaseRestHandler { if (source != null) { countRequest.query(source); } else { - byte[] querySource = RestActions.parseQuerySource(request); + BytesStream querySource = RestActions.parseQuerySource(request); if (querySource != null) { - countRequest.query(querySource); + countRequest.query(querySource.underlyingBytes(), 0, querySource.size(), false); } } } diff --git a/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java b/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java index 81588388110..0e5107e25a6 100644 --- a/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java +++ b/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.deletebyquery.ShardDeleteByQueryRequest; import org.elasticsearch.action.support.replication.ReplicationType; import org.elasticsearch.client.Client; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.BytesStream; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.*; @@ -67,7 +68,8 @@ public class RestDeleteByQueryAction extends BaseRestHandler { if (source != null) { deleteByQueryRequest.query(source); } else { - deleteByQueryRequest.query(RestActions.parseQuerySource(request)); + BytesStream bytes = RestActions.parseQuerySource(request); + deleteByQueryRequest.query(bytes.underlyingBytes(), 0, bytes.size(), false); } } deleteByQueryRequest.types(splitTypes(request.param("type"))); diff --git a/src/main/java/org/elasticsearch/rest/action/support/RestActions.java b/src/main/java/org/elasticsearch/rest/action/support/RestActions.java index ae4bd7b8db9..b74b9d15c84 100644 --- a/src/main/java/org/elasticsearch/rest/action/support/RestActions.java +++ b/src/main/java/org/elasticsearch/rest/action/support/RestActions.java @@ -23,6 +23,7 @@ import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastOperationResponse; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.BytesStream; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryStringQueryBuilder; @@ -69,7 +70,7 @@ public class RestActions { builder.endObject(); } - public static byte[] parseQuerySource(RestRequest request) { + public static BytesStream parseQuerySource(RestRequest request) { String queryString = request.param("q"); if (queryString == null) { return null; diff --git a/src/test/java/org/elasticsearch/test/unit/index/query/SimpleIndexQueryParserTests.java b/src/test/java/org/elasticsearch/test/unit/index/query/SimpleIndexQueryParserTests.java index ec90f20f5c0..0137c06a735 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/query/SimpleIndexQueryParserTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/query/SimpleIndexQueryParserTests.java @@ -288,7 +288,7 @@ public class SimpleIndexQueryParserTests { @Test public void testTermQueryBuilder() throws IOException { IndexQueryParserService queryParser = queryParser(); - Query parsedQuery = queryParser.parse(termQuery("age", 34).buildAsBytes()).query(); + Query parsedQuery = queryParser.parse(termQuery("age", 34).buildAsBytes().copiedByteArray()).query(); assertThat(parsedQuery, instanceOf(NumericRangeQuery.class)); NumericRangeQuery fieldQuery = (NumericRangeQuery) parsedQuery; assertThat(fieldQuery.getMin().intValue(), equalTo(34)); @@ -313,7 +313,7 @@ public class SimpleIndexQueryParserTests { @Test public void testFuzzyQueryBuilder() throws IOException { IndexQueryParserService queryParser = queryParser(); - Query parsedQuery = queryParser.parse(fuzzyQuery("name.first", "sh").buildAsBytes()).query(); + Query parsedQuery = queryParser.parse(fuzzyQuery("name.first", "sh").buildAsBytes().copiedByteArray()).query(); assertThat(parsedQuery, instanceOf(FuzzyQuery.class)); FuzzyQuery fuzzyQuery = (FuzzyQuery) parsedQuery; assertThat(fuzzyQuery.getTerm(), equalTo(new Term("name.first", "sh"))); @@ -332,7 +332,7 @@ public class SimpleIndexQueryParserTests { @Test public void testFuzzyQueryWithFieldsBuilder() throws IOException { IndexQueryParserService queryParser = queryParser(); - Query parsedQuery = queryParser.parse(fuzzyQuery("name.first", "sh").minSimilarity(0.1f).prefixLength(1).boost(2.0f).buildAsBytes()).query(); + Query parsedQuery = queryParser.parse(fuzzyQuery("name.first", "sh").minSimilarity(0.1f).prefixLength(1).boost(2.0f).buildAsBytes().copiedByteArray()).query(); assertThat(parsedQuery, instanceOf(FuzzyQuery.class)); FuzzyQuery fuzzyQuery = (FuzzyQuery) parsedQuery; assertThat(fuzzyQuery.getTerm(), equalTo(new Term("name.first", "sh"))); @@ -368,7 +368,7 @@ public class SimpleIndexQueryParserTests { @Test public void testFieldQueryBuilder1() throws IOException { IndexQueryParserService queryParser = queryParser(); - Query parsedQuery = queryParser.parse(fieldQuery("age", 34).buildAsBytes()).query(); + Query parsedQuery = queryParser.parse(fieldQuery("age", 34).buildAsBytes().copiedByteArray()).query(); assertThat(parsedQuery, instanceOf(NumericRangeQuery.class)); NumericRangeQuery fieldQuery = (NumericRangeQuery) parsedQuery; assertThat(fieldQuery.getMin().intValue(), equalTo(34)); From f1f2fb2ba79b98200013b970cc0a5db721068e75 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 9 Jan 2012 15:24:17 +0200 Subject: [PATCH 146/270] Nodes Info API: Allow to specify which info to get back, simpler URI paths, closes #1596. --- .../admin/cluster/node/info/NodeInfo.java | 66 +++++-- .../cluster/node/info/NodesInfoRequest.java | 157 +++++++++++++++- .../cluster/node/info/NodesInfoResponse.java | 76 +++++++- .../node/info/TransportNodesInfoAction.java | 32 +++- .../node/info/NodesInfoRequestBuilder.java | 65 +++++++ .../node/service/NodeService.java | 16 +- .../node/info/RestNodesInfoAction.java | 169 +++++++++++------- 7 files changed, 495 insertions(+), 86 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java index 07b4e3f6f5a..f282d2d47d1 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java @@ -42,31 +42,39 @@ import java.util.Map; */ public class NodeInfo extends NodeOperationResponse { + @Nullable private ImmutableMap serviceAttributes; @Nullable private String hostname; + @Nullable private Settings settings; + @Nullable private OsInfo os; + @Nullable private ProcessInfo process; + @Nullable private JvmInfo jvm; + @Nullable private NetworkInfo network; + @Nullable private TransportInfo transport; + @Nullable private HttpInfo http; NodeInfo() { } - public NodeInfo(@Nullable String hostname, DiscoveryNode node, ImmutableMap serviceAttributes, Settings settings, - OsInfo os, ProcessInfo process, JvmInfo jvm, NetworkInfo network, - TransportInfo transport, @Nullable HttpInfo http) { + public NodeInfo(@Nullable String hostname, DiscoveryNode node, @Nullable ImmutableMap serviceAttributes, @Nullable Settings settings, + @Nullable OsInfo os, @Nullable ProcessInfo process, @Nullable JvmInfo jvm, @Nullable NetworkInfo network, + @Nullable TransportInfo transport, @Nullable HttpInfo http) { super(node); this.hostname = hostname; this.serviceAttributes = serviceAttributes; @@ -98,6 +106,7 @@ public class NodeInfo extends NodeOperationResponse { /** * The service attributes of the node. */ + @Nullable public ImmutableMap serviceAttributes() { return this.serviceAttributes; } @@ -105,6 +114,7 @@ public class NodeInfo extends NodeOperationResponse { /** * The attributes of the node. */ + @Nullable public ImmutableMap getServiceAttributes() { return serviceAttributes(); } @@ -112,6 +122,7 @@ public class NodeInfo extends NodeOperationResponse { /** * The settings of the node. */ + @Nullable public Settings settings() { return this.settings; } @@ -119,6 +130,7 @@ public class NodeInfo extends NodeOperationResponse { /** * The settings of the node. */ + @Nullable public Settings getSettings() { return settings(); } @@ -126,6 +138,7 @@ public class NodeInfo extends NodeOperationResponse { /** * Operating System level information. */ + @Nullable public OsInfo os() { return this.os; } @@ -133,6 +146,7 @@ public class NodeInfo extends NodeOperationResponse { /** * Operating System level information. */ + @Nullable public OsInfo getOs() { return os(); } @@ -140,6 +154,7 @@ public class NodeInfo extends NodeOperationResponse { /** * Process level information. */ + @Nullable public ProcessInfo process() { return process; } @@ -147,6 +162,7 @@ public class NodeInfo extends NodeOperationResponse { /** * Process level information. */ + @Nullable public ProcessInfo getProcess() { return process(); } @@ -154,6 +170,7 @@ public class NodeInfo extends NodeOperationResponse { /** * JVM level information. */ + @Nullable public JvmInfo jvm() { return jvm; } @@ -161,6 +178,7 @@ public class NodeInfo extends NodeOperationResponse { /** * JVM level information. */ + @Nullable public JvmInfo getJvm() { return jvm(); } @@ -168,6 +186,7 @@ public class NodeInfo extends NodeOperationResponse { /** * Network level information. */ + @Nullable public NetworkInfo network() { return network; } @@ -175,22 +194,27 @@ public class NodeInfo extends NodeOperationResponse { /** * Network level information. */ + @Nullable public NetworkInfo getNetwork() { return network(); } + @Nullable public TransportInfo transport() { return transport; } + @Nullable public TransportInfo getTransport() { return transport(); } + @Nullable public HttpInfo http() { return http; } + @Nullable public HttpInfo getHttp() { return http(); } @@ -207,13 +231,17 @@ public class NodeInfo extends NodeOperationResponse { if (in.readBoolean()) { hostname = in.readUTF(); } - ImmutableMap.Builder builder = ImmutableMap.builder(); - int size = in.readVInt(); - for (int i = 0; i < size; i++) { - builder.put(in.readUTF(), in.readUTF()); + if (in.readBoolean()) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + int size = in.readVInt(); + for (int i = 0; i < size; i++) { + builder.put(in.readUTF(), in.readUTF()); + } + serviceAttributes = builder.build(); + } + if (in.readBoolean()) { + settings = ImmutableSettings.readSettingsFromStream(in); } - serviceAttributes = builder.build(); - settings = ImmutableSettings.readSettingsFromStream(in); if (in.readBoolean()) { os = OsInfo.readOsInfo(in); } @@ -243,12 +271,22 @@ public class NodeInfo extends NodeOperationResponse { out.writeBoolean(true); out.writeUTF(hostname); } - out.writeVInt(serviceAttributes.size()); - for (Map.Entry entry : serviceAttributes.entrySet()) { - out.writeUTF(entry.getKey()); - out.writeUTF(entry.getValue()); + if (serviceAttributes() == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeVInt(serviceAttributes.size()); + for (Map.Entry entry : serviceAttributes.entrySet()) { + out.writeUTF(entry.getKey()); + out.writeUTF(entry.getValue()); + } + } + if (settings == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + ImmutableSettings.writeSettingsToStream(settings, out); } - ImmutableSettings.writeSettingsToStream(settings, out); if (os == null) { out.writeBoolean(false); } else { diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java index b49b427069e..fdd033ace03 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java @@ -20,14 +20,24 @@ package org.elasticsearch.action.admin.cluster.node.info; import org.elasticsearch.action.support.nodes.NodesOperationRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; /** * A request to get node (cluster) level information. - * - * */ public class NodesInfoRequest extends NodesOperationRequest { + private boolean settings = false; + private boolean os = false; + private boolean process = false; + private boolean jvm = false; + private boolean network = false; + private boolean transport = false; + private boolean http = false; + public NodesInfoRequest() { } @@ -38,4 +48,147 @@ public class NodesInfoRequest extends NodesOperationRequest { public NodesInfoRequest(String... nodesIds) { super(nodesIds); } + + /** + * Clears all info flags. + */ + public NodesInfoRequest clear() { + settings = false; + os = false; + process = false; + jvm = false; + network = false; + transport = false; + http = false; + return this; + } + + /** + * Should the node settings be returned. + */ + public boolean settings() { + return this.settings; + } + + /** + * Should the node settings be returned. + */ + public NodesInfoRequest settings(boolean settings) { + this.settings = settings; + return this; + } + + /** + * Should the node OS be returned. + */ + public boolean os() { + return this.os; + } + + /** + * Should the node OS be returned. + */ + public NodesInfoRequest os(boolean os) { + this.os = os; + return this; + } + + /** + * Should the node Process be returned. + */ + public boolean process() { + return this.process; + } + + /** + * Should the node Process be returned. + */ + public NodesInfoRequest process(boolean process) { + this.process = process; + return this; + } + + /** + * Should the node JVM be returned. + */ + public boolean jvm() { + return this.jvm; + } + + /** + * Should the node JVM be returned. + */ + public NodesInfoRequest jvm(boolean jvm) { + this.jvm = jvm; + return this; + } + + /** + * Should the node Network be returned. + */ + public boolean network() { + return this.network; + } + + /** + * Should the node Network be returned. + */ + public NodesInfoRequest network(boolean network) { + this.network = network; + return this; + } + + /** + * Should the node Transport be returned. + */ + public boolean transport() { + return this.transport; + } + + /** + * Should the node Transport be returned. + */ + public NodesInfoRequest transport(boolean transport) { + this.transport = transport; + return this; + } + + /** + * Should the node HTTP be returned. + */ + public boolean http() { + return this.http; + } + + /** + * Should the node HTTP be returned. + */ + public NodesInfoRequest http(boolean http) { + this.http = http; + return this; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + settings = in.readBoolean(); + os = in.readBoolean(); + process = in.readBoolean(); + jvm = in.readBoolean(); + network = in.readBoolean(); + transport = in.readBoolean(); + http = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBoolean(settings); + out.writeBoolean(os); + out.writeBoolean(process); + out.writeBoolean(jvm); + out.writeBoolean(network); + out.writeBoolean(transport); + out.writeBoolean(http); + } } diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java index b942104837a..f8857db44d8 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java @@ -23,13 +23,20 @@ import org.elasticsearch.action.support.nodes.NodesOperationResponse; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; /** * */ -public class NodesInfoResponse extends NodesOperationResponse { +public class NodesInfoResponse extends NodesOperationResponse implements ToXContent { + + private SettingsFilter settingsFilter; public NodesInfoResponse() { } @@ -55,4 +62,71 @@ public class NodesInfoResponse extends NodesOperationResponse { node.writeTo(out); } } + + public NodesInfoResponse settingsFilter(SettingsFilter settingsFilter) { + this.settingsFilter = settingsFilter; + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("cluster_name", clusterName().value()); + + builder.startObject("nodes"); + for (NodeInfo nodeInfo : this) { + builder.startObject(nodeInfo.node().id(), XContentBuilder.FieldCaseConversion.NONE); + + builder.field("name", nodeInfo.node().name(), XContentBuilder.FieldCaseConversion.NONE); + builder.field("transport_address", nodeInfo.node().address().toString()); + + if (nodeInfo.hostname() != null) { + builder.field("hostname", nodeInfo.hostname(), XContentBuilder.FieldCaseConversion.NONE); + } + + if (nodeInfo.serviceAttributes() != null) { + for (Map.Entry nodeAttribute : nodeInfo.serviceAttributes().entrySet()) { + builder.field(nodeAttribute.getKey(), nodeAttribute.getValue()); + } + } + + builder.startObject("attributes"); + for (Map.Entry attr : nodeInfo.node().attributes().entrySet()) { + builder.field(attr.getKey(), attr.getValue()); + } + builder.endObject(); + + + if (nodeInfo.settings() != null) { + builder.startObject("settings"); + Settings settings = settingsFilter.filterSettings(nodeInfo.settings()); + for (Map.Entry entry : settings.getAsMap().entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + builder.endObject(); + } + + if (nodeInfo.os() != null) { + nodeInfo.os().toXContent(builder, params); + } + if (nodeInfo.process() != null) { + nodeInfo.process().toXContent(builder, params); + } + if (nodeInfo.jvm() != null) { + nodeInfo.jvm().toXContent(builder, params); + } + if (nodeInfo.network() != null) { + nodeInfo.network().toXContent(builder, params); + } + if (nodeInfo.transport() != null) { + nodeInfo.transport().toXContent(builder, params); + } + if (nodeInfo.http() != null) { + nodeInfo.http().toXContent(builder, params); + } + + builder.endObject(); + } + builder.endObject(); + return builder; + } } diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java index 80d335ad745..ddca44262d5 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java @@ -26,11 +26,14 @@ import org.elasticsearch.action.support.nodes.TransportNodesOperationAction; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.service.NodeService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -84,7 +87,7 @@ public class TransportNodesInfoAction extends TransportNodesOperationAction listener) { client.nodesInfo(request, listener); diff --git a/src/main/java/org/elasticsearch/node/service/NodeService.java b/src/main/java/org/elasticsearch/node/service/NodeService.java index 74955db9f47..9e456d7ac65 100644 --- a/src/main/java/org/elasticsearch/node/service/NodeService.java +++ b/src/main/java/org/elasticsearch/node/service/NodeService.java @@ -19,8 +19,6 @@ package org.elasticsearch.node.service; -import java.net.InetAddress; - import com.google.common.collect.ImmutableMap; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; @@ -37,6 +35,8 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.monitor.MonitorService; import org.elasticsearch.transport.TransportService; +import java.net.InetAddress; + /** */ public class NodeService extends AbstractComponent { @@ -107,6 +107,18 @@ public class NodeService extends AbstractComponent { transportService.info(), httpServer == null ? null : httpServer.info()); } + public NodeInfo info(boolean settings, boolean os, boolean process, boolean jvm, boolean network, boolean transport, boolean http) { + return new NodeInfo(hostname, clusterService.state().nodes().localNode(), serviceAttributes, + settings ? this.settings : null, + os ? monitorService.osService().info() : null, + process ? monitorService.processService().info() : null, + jvm ? monitorService.jvmService().info() : null, + network ? monitorService.networkService().info() : null, + transport ? transportService.info() : null, + http ? (httpServer == null ? null : httpServer.info()) : null + ); + } + public NodeStats stats() { // for indices stats we want to include previous allocated shards stats as well (it will // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats) diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java index 22d3a0ac3d8..5fc21628273 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.rest.action.admin.cluster.node.info; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.client.Client; @@ -33,7 +32,6 @@ import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestXContentBuilder; import java.io.IOException; -import java.util.Map; /** * @@ -48,6 +46,29 @@ public class RestNodesInfoAction extends BaseRestHandler { super(settings, client); controller.registerHandler(RestRequest.Method.GET, "/_cluster/nodes", this); controller.registerHandler(RestRequest.Method.GET, "/_cluster/nodes/{nodeId}", this); + controller.registerHandler(RestRequest.Method.GET, "/_nodes", this); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}", this); + + controller.registerHandler(RestRequest.Method.GET, "/_nodes/settings", new RestSettingsHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/settings", new RestSettingsHandler()); + + controller.registerHandler(RestRequest.Method.GET, "/_nodes/os", new RestOsHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/os", new RestOsHandler()); + + controller.registerHandler(RestRequest.Method.GET, "/_nodes/process", new RestProcessHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/process", new RestProcessHandler()); + + controller.registerHandler(RestRequest.Method.GET, "/_nodes/jvm", new RestJvmHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/jvm", new RestJvmHandler()); + + controller.registerHandler(RestRequest.Method.GET, "/_nodes/network", new RestNetworkHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/network", new RestNetworkHandler()); + + controller.registerHandler(RestRequest.Method.GET, "/_nodes/transport", new RestTransportHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/transport", new RestTransportHandler()); + + controller.registerHandler(RestRequest.Method.GET, "/_nodes/http", new RestHttpHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/http", new RestHttpHandler()); this.settingsFilter = settingsFilter; } @@ -55,71 +76,34 @@ public class RestNodesInfoAction extends BaseRestHandler { @Override public void handleRequest(final RestRequest request, final RestChannel channel) { String[] nodesIds = RestActions.splitNodes(request.param("nodeId")); - final boolean includeSettings = request.paramAsBoolean("settings", false); - NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(nodesIds); + final NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(nodesIds); + + boolean clear = request.paramAsBoolean("clear", false); + if (clear) { + nodesInfoRequest.clear(); + } + nodesInfoRequest.settings(request.paramAsBoolean("settings", nodesInfoRequest.settings())); + nodesInfoRequest.os(request.paramAsBoolean("os", nodesInfoRequest.os())); + nodesInfoRequest.process(request.paramAsBoolean("process", nodesInfoRequest.process())); + nodesInfoRequest.jvm(request.paramAsBoolean("jvm", nodesInfoRequest.jvm())); + nodesInfoRequest.network(request.paramAsBoolean("network", nodesInfoRequest.network())); + nodesInfoRequest.transport(request.paramAsBoolean("transport", nodesInfoRequest.transport())); + nodesInfoRequest.http(request.paramAsBoolean("http", nodesInfoRequest.http())); + + executeNodeRequest(request, channel, nodesInfoRequest); + } + + void executeNodeRequest(final RestRequest request, final RestChannel channel, NodesInfoRequest nodesInfoRequest) { nodesInfoRequest.listenerThreaded(false); client.admin().cluster().nodesInfo(nodesInfoRequest, new ActionListener() { @Override - public void onResponse(NodesInfoResponse result) { + public void onResponse(NodesInfoResponse response) { try { + response.settingsFilter(settingsFilter); XContentBuilder builder = RestXContentBuilder.restContentBuilder(request); builder.startObject(); - builder.field("cluster_name", result.clusterName().value()); - - builder.startObject("nodes"); - for (NodeInfo nodeInfo : result) { - builder.startObject(nodeInfo.node().id(), XContentBuilder.FieldCaseConversion.NONE); - - builder.field("name", nodeInfo.node().name(), XContentBuilder.FieldCaseConversion.NONE); - builder.field("transport_address", nodeInfo.node().address().toString()); - - if (nodeInfo.hostname() != null) { - builder.field("hostname", nodeInfo.hostname(), XContentBuilder.FieldCaseConversion.NONE); - } - - for (Map.Entry nodeAttribute : nodeInfo.serviceAttributes().entrySet()) { - builder.field(nodeAttribute.getKey(), nodeAttribute.getValue()); - } - - builder.startObject("attributes"); - for (Map.Entry attr : nodeInfo.node().attributes().entrySet()) { - builder.field(attr.getKey(), attr.getValue()); - } - builder.endObject(); - - - if (includeSettings) { - builder.startObject("settings"); - Settings settings = settingsFilter.filterSettings(nodeInfo.settings()); - for (Map.Entry entry : settings.getAsMap().entrySet()) { - builder.field(entry.getKey(), entry.getValue()); - } - builder.endObject(); - } - - if (nodeInfo.os() != null) { - nodeInfo.os().toXContent(builder, request); - } - if (nodeInfo.process() != null) { - nodeInfo.process().toXContent(builder, request); - } - if (nodeInfo.jvm() != null) { - nodeInfo.jvm().toXContent(builder, request); - } - if (nodeInfo.network() != null) { - nodeInfo.network().toXContent(builder, request); - } - if (nodeInfo.transport() != null) { - nodeInfo.transport().toXContent(builder, request); - } - if (nodeInfo.http() != null) { - nodeInfo.http().toXContent(builder, request); - } - - builder.endObject(); - } - builder.endObject(); - + builder.field("ok", true); + response.toXContent(builder, request); builder.endObject(); channel.sendResponse(new XContentRestResponse(request, RestStatus.OK, builder)); } catch (Exception e) { @@ -137,4 +121,67 @@ public class RestNodesInfoAction extends BaseRestHandler { } }); } + + class RestSettingsHandler implements RestHandler { + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(RestActions.splitNodes(request.param("nodeId"))); + nodesInfoRequest.clear().settings(true); + executeNodeRequest(request, channel, nodesInfoRequest); + } + } + + class RestOsHandler implements RestHandler { + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(RestActions.splitNodes(request.param("nodeId"))); + nodesInfoRequest.clear().os(true); + executeNodeRequest(request, channel, nodesInfoRequest); + } + } + + class RestProcessHandler implements RestHandler { + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(RestActions.splitNodes(request.param("nodeId"))); + nodesInfoRequest.clear().process(true); + executeNodeRequest(request, channel, nodesInfoRequest); + } + } + + class RestJvmHandler implements RestHandler { + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(RestActions.splitNodes(request.param("nodeId"))); + nodesInfoRequest.clear().jvm(true); + executeNodeRequest(request, channel, nodesInfoRequest); + } + } + + class RestNetworkHandler implements RestHandler { + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(RestActions.splitNodes(request.param("nodeId"))); + nodesInfoRequest.clear().network(true); + executeNodeRequest(request, channel, nodesInfoRequest); + } + } + + class RestTransportHandler implements RestHandler { + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(RestActions.splitNodes(request.param("nodeId"))); + nodesInfoRequest.clear().transport(true); + executeNodeRequest(request, channel, nodesInfoRequest); + } + } + + class RestHttpHandler implements RestHandler { + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(RestActions.splitNodes(request.param("nodeId"))); + nodesInfoRequest.clear().http(true); + executeNodeRequest(request, channel, nodesInfoRequest); + } + } } From 4464fe1dc172f176862caf9d907042d92d9636ee Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 9 Jan 2012 18:01:41 +0200 Subject: [PATCH 147/270] Node Stats API: Add specific flags for stats, simplified REST paths, closes #1597. --- .../cluster/node/info/NodesInfoResponse.java | 10 +- .../admin/cluster/node/stats/NodeStats.java | 52 +++++- .../cluster/node/stats/NodesStatsRequest.java | 158 +++++++++++++++++- .../node/stats/NodesStatsResponse.java | 58 ++++++- .../node/stats/TransportNodesStatsAction.java | 32 +++- .../node/stats/NodesStatsRequestBuilder.java | 64 +++++++ .../node/service/NodeService.java | 15 +- .../node/stats/RestNodesStatsAction.java | 141 ++++++++++++---- 8 files changed, 474 insertions(+), 56 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java index f8857db44d8..8b6fc920eb0 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java @@ -89,11 +89,13 @@ public class NodesInfoResponse extends NodesOperationResponse implemen } } - builder.startObject("attributes"); - for (Map.Entry attr : nodeInfo.node().attributes().entrySet()) { - builder.field(attr.getKey(), attr.getValue()); + if (!nodeInfo.node().attributes().isEmpty()) { + builder.startObject("attributes"); + for (Map.Entry attr : nodeInfo.node().attributes().entrySet()) { + builder.field(attr.getKey(), attr.getValue()); + } + builder.endObject(); } - builder.endObject(); if (nodeInfo.settings() != null) { diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java index f7158fc668c..85008b36554 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java @@ -36,32 +36,41 @@ import java.io.IOException; /** * Node statistics (static, does not change over time). - * - * */ public class NodeStats extends NodeOperationResponse { + @Nullable + private String hostname; + + @Nullable private NodeIndicesStats indices; + @Nullable private OsStats os; + @Nullable private ProcessStats process; + @Nullable private JvmStats jvm; + @Nullable private NetworkStats network; + @Nullable private TransportStats transport; + @Nullable private HttpStats http; NodeStats() { } - public NodeStats(DiscoveryNode node, NodeIndicesStats indices, - OsStats os, ProcessStats process, JvmStats jvm, NetworkStats network, - TransportStats transport, @Nullable HttpStats http) { + public NodeStats(DiscoveryNode node, @Nullable String hostname, @Nullable NodeIndicesStats indices, + @Nullable OsStats os, @Nullable ProcessStats process, @Nullable JvmStats jvm, @Nullable NetworkStats network, + @Nullable TransportStats transport, @Nullable HttpStats http) { super(node); + this.hostname = hostname; this.indices = indices; this.os = os; this.process = process; @@ -71,9 +80,20 @@ public class NodeStats extends NodeOperationResponse { this.http = http; } + @Nullable + public String hostname() { + return this.hostname; + } + + @Nullable + public String getHostname() { + return this.hostname; + } + /** * Indices level stats. */ + @Nullable public NodeIndicesStats indices() { return this.indices; } @@ -81,6 +101,7 @@ public class NodeStats extends NodeOperationResponse { /** * Indices level stats. */ + @Nullable public NodeIndicesStats getIndices() { return indices(); } @@ -88,6 +109,7 @@ public class NodeStats extends NodeOperationResponse { /** * Operating System level statistics. */ + @Nullable public OsStats os() { return this.os; } @@ -95,6 +117,7 @@ public class NodeStats extends NodeOperationResponse { /** * Operating System level statistics. */ + @Nullable public OsStats getOs() { return os(); } @@ -102,6 +125,7 @@ public class NodeStats extends NodeOperationResponse { /** * Process level statistics. */ + @Nullable public ProcessStats process() { return process; } @@ -109,6 +133,7 @@ public class NodeStats extends NodeOperationResponse { /** * Process level statistics. */ + @Nullable public ProcessStats getProcess() { return process(); } @@ -116,6 +141,7 @@ public class NodeStats extends NodeOperationResponse { /** * JVM level statistics. */ + @Nullable public JvmStats jvm() { return jvm; } @@ -123,6 +149,7 @@ public class NodeStats extends NodeOperationResponse { /** * JVM level statistics. */ + @Nullable public JvmStats getJvm() { return jvm(); } @@ -130,6 +157,7 @@ public class NodeStats extends NodeOperationResponse { /** * Network level statistics. */ + @Nullable public NetworkStats network() { return network; } @@ -137,22 +165,27 @@ public class NodeStats extends NodeOperationResponse { /** * Network level statistics. */ + @Nullable public NetworkStats getNetwork() { return network(); } + @Nullable public TransportStats transport() { return this.transport; } + @Nullable public TransportStats getTransport() { return transport(); } + @Nullable public HttpStats http() { return this.http; } + @Nullable public HttpStats getHttp() { return http(); } @@ -166,6 +199,9 @@ public class NodeStats extends NodeOperationResponse { @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); + if (in.readBoolean()) { + hostname = in.readUTF(); + } if (in.readBoolean()) { indices = NodeIndicesStats.readIndicesStats(in); } @@ -192,6 +228,12 @@ public class NodeStats extends NodeOperationResponse { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); + if (hostname == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeUTF(hostname); + } if (indices == null) { out.writeBoolean(false); } else { diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java index 42fbf59fc8c..5cefb6208c3 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -20,14 +20,24 @@ package org.elasticsearch.action.admin.cluster.node.stats; import org.elasticsearch.action.support.nodes.NodesOperationRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; /** * A request to get node (cluster) level stats. - * - * */ public class NodesStatsRequest extends NodesOperationRequest { + private boolean indices = true; + private boolean os; + private boolean process; + private boolean jvm; + private boolean network; + private boolean transport; + private boolean http; + protected NodesStatsRequest() { } @@ -38,4 +48,148 @@ public class NodesStatsRequest extends NodesOperationRequest { public NodesStatsRequest(String... nodesIds) { super(nodesIds); } + + /** + * Clears all the request flags. + */ + public NodesStatsRequest clear() { + this.indices = false; + this.os = false; + this.process = false; + this.jvm = false; + this.network = false; + this.transport = false; + this.http = false; + return this; + } + + /** + * Should indices stats be returned. + */ + public boolean indices() { + return this.indices; + } + + /** + * Should indices stats be returned. + */ + public NodesStatsRequest indices(boolean indices) { + this.indices = indices; + return this; + } + + /** + * Should the node OS be returned. + */ + public boolean os() { + return this.os; + } + + /** + * Should the node OS be returned. + */ + public NodesStatsRequest os(boolean os) { + this.os = os; + return this; + } + + /** + * Should the node Process be returned. + */ + public boolean process() { + return this.process; + } + + /** + * Should the node Process be returned. + */ + public NodesStatsRequest process(boolean process) { + this.process = process; + return this; + } + + /** + * Should the node JVM be returned. + */ + public boolean jvm() { + return this.jvm; + } + + /** + * Should the node JVM be returned. + */ + public NodesStatsRequest jvm(boolean jvm) { + this.jvm = jvm; + return this; + } + + /** + * Should the node Network be returned. + */ + public boolean network() { + return this.network; + } + + /** + * Should the node Network be returned. + */ + public NodesStatsRequest network(boolean network) { + this.network = network; + return this; + } + + /** + * Should the node Transport be returned. + */ + public boolean transport() { + return this.transport; + } + + /** + * Should the node Transport be returned. + */ + public NodesStatsRequest transport(boolean transport) { + this.transport = transport; + return this; + } + + /** + * Should the node HTTP be returned. + */ + public boolean http() { + return this.http; + } + + /** + * Should the node HTTP be returned. + */ + public NodesStatsRequest http(boolean http) { + this.http = http; + return this; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + indices = in.readBoolean(); + os = in.readBoolean(); + process = in.readBoolean(); + jvm = in.readBoolean(); + network = in.readBoolean(); + transport = in.readBoolean(); + http = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBoolean(indices); + out.writeBoolean(os); + out.writeBoolean(process); + out.writeBoolean(jvm); + out.writeBoolean(network); + out.writeBoolean(transport); + out.writeBoolean(http); + } + } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java index 39d451b21b3..3db6edd9830 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java @@ -23,13 +23,16 @@ import org.elasticsearch.action.support.nodes.NodesOperationResponse; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; /** * */ -public class NodesStatsResponse extends NodesOperationResponse { +public class NodesStatsResponse extends NodesOperationResponse implements ToXContent { NodesStatsResponse() { } @@ -55,4 +58,57 @@ public class NodesStatsResponse extends NodesOperationResponse { node.writeTo(out); } } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("cluster_name", clusterName().value()); + + builder.startObject("nodes"); + for (NodeStats nodeStats : this) { + builder.startObject(nodeStats.node().id(), XContentBuilder.FieldCaseConversion.NONE); + + builder.field("name", nodeStats.node().name(), XContentBuilder.FieldCaseConversion.NONE); + builder.field("transport_address", nodeStats.node().address().toString()); + + if (nodeStats.hostname() != null) { + builder.field("hostname", nodeStats.hostname(), XContentBuilder.FieldCaseConversion.NONE); + } + + if (!nodeStats.node().attributes().isEmpty()) { + builder.startObject("attributes"); + for (Map.Entry attr : nodeStats.node().attributes().entrySet()) { + builder.field(attr.getKey(), attr.getValue()); + } + builder.endObject(); + } + + if (nodeStats.indices() != null) { + nodeStats.indices().toXContent(builder, params); + } + + if (nodeStats.os() != null) { + nodeStats.os().toXContent(builder, params); + } + if (nodeStats.process() != null) { + nodeStats.process().toXContent(builder, params); + } + if (nodeStats.jvm() != null) { + nodeStats.jvm().toXContent(builder, params); + } + if (nodeStats.network() != null) { + nodeStats.network().toXContent(builder, params); + } + if (nodeStats.transport() != null) { + nodeStats.transport().toXContent(builder, params); + } + if (nodeStats.http() != null) { + nodeStats.http().toXContent(builder, params); + } + + builder.endObject(); + } + builder.endObject(); + + return builder; + } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index 6a16c298404..3ccb8a5dc65 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -27,11 +27,14 @@ import org.elasticsearch.action.support.nodes.TransportNodesOperationAction; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.service.NodeService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; import java.util.List; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -84,7 +87,7 @@ public class TransportNodesStatsAction extends TransportNodesOperationAction listener) { client.nodesStats(request, listener); diff --git a/src/main/java/org/elasticsearch/node/service/NodeService.java b/src/main/java/org/elasticsearch/node/service/NodeService.java index 9e456d7ac65..37f94a44482 100644 --- a/src/main/java/org/elasticsearch/node/service/NodeService.java +++ b/src/main/java/org/elasticsearch/node/service/NodeService.java @@ -122,9 +122,22 @@ public class NodeService extends AbstractComponent { public NodeStats stats() { // for indices stats we want to include previous allocated shards stats as well (it will // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats) - return new NodeStats(clusterService.state().nodes().localNode(), indicesService.stats(true), + return new NodeStats(clusterService.state().nodes().localNode(), hostname, indicesService.stats(true), monitorService.osService().stats(), monitorService.processService().stats(), monitorService.jvmService().stats(), monitorService.networkService().stats(), transportService.stats(), httpServer == null ? null : httpServer.stats()); } + + public NodeStats stats(boolean indices, boolean os, boolean process, boolean jvm, boolean network, boolean transport, boolean http) { + // for indices stats we want to include previous allocated shards stats as well (it will + // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats) + return new NodeStats(clusterService.state().nodes().localNode(), hostname, + indices ? indicesService.stats(true) : null, + os ? monitorService.osService().stats() : null, + process ? monitorService.processService().stats() : null, + jvm ? monitorService.jvmService().stats() : null, + network ? monitorService.networkService().stats() : null, + transport ? transportService.stats() : null, + http ? (httpServer == null ? null : httpServer.stats()) : null); + } } diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java index 19d5ecf45fe..5aed0920645 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.rest.action.admin.cluster.node.stats; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.client.Client; @@ -43,54 +42,59 @@ public class RestNodesStatsAction extends BaseRestHandler { super(settings, client); controller.registerHandler(RestRequest.Method.GET, "/_cluster/nodes/stats", this); controller.registerHandler(RestRequest.Method.GET, "/_cluster/nodes/{nodeId}/stats", this); + + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats", this); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats", this); + + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/indices", new RestIndicesHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/indices", new RestIndicesHandler()); + + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/os", new RestOsHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/os", new RestOsHandler()); + + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/process", new RestProcessHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/process", new RestProcessHandler()); + + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/jvm", new RestJvmHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/jvm", new RestJvmHandler()); + + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/network", new RestNetworkHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/network", new RestNetworkHandler()); + + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/transport", new RestTransportHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/transport", new RestTransportHandler()); + + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/http", new RestHttpHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/http", new RestHttpHandler()); } @Override public void handleRequest(final RestRequest request, final RestChannel channel) { String[] nodesIds = RestActions.splitNodes(request.param("nodeId")); NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(nodesIds); + boolean clear = request.paramAsBoolean("clear", false); + if (clear) { + nodesStatsRequest.clear(); + } + nodesStatsRequest.indices(request.paramAsBoolean("indices", nodesStatsRequest.indices())); + nodesStatsRequest.os(request.paramAsBoolean("os", nodesStatsRequest.os())); + nodesStatsRequest.process(request.paramAsBoolean("process", nodesStatsRequest.process())); + nodesStatsRequest.jvm(request.paramAsBoolean("jvm", nodesStatsRequest.jvm())); + nodesStatsRequest.network(request.paramAsBoolean("network", nodesStatsRequest.network())); + nodesStatsRequest.transport(request.paramAsBoolean("transport", nodesStatsRequest.transport())); + nodesStatsRequest.http(request.paramAsBoolean("http", nodesStatsRequest.http())); + executeNodeStats(request, channel, nodesStatsRequest); + } + + void executeNodeStats(final RestRequest request, final RestChannel channel, final NodesStatsRequest nodesStatsRequest) { nodesStatsRequest.listenerThreaded(false); client.admin().cluster().nodesStats(nodesStatsRequest, new ActionListener() { @Override - public void onResponse(NodesStatsResponse result) { + public void onResponse(NodesStatsResponse response) { try { XContentBuilder builder = RestXContentBuilder.restContentBuilder(request); builder.startObject(); - builder.field("cluster_name", result.clusterName().value()); - - builder.startObject("nodes"); - for (NodeStats nodeStats : result) { - builder.startObject(nodeStats.node().id(), XContentBuilder.FieldCaseConversion.NONE); - - builder.field("name", nodeStats.node().name(), XContentBuilder.FieldCaseConversion.NONE); - - if (nodeStats.indices() != null) { - nodeStats.indices().toXContent(builder, request); - } - - if (nodeStats.os() != null) { - nodeStats.os().toXContent(builder, request); - } - if (nodeStats.process() != null) { - nodeStats.process().toXContent(builder, request); - } - if (nodeStats.jvm() != null) { - nodeStats.jvm().toXContent(builder, request); - } - if (nodeStats.network() != null) { - nodeStats.network().toXContent(builder, request); - } - if (nodeStats.transport() != null) { - nodeStats.transport().toXContent(builder, request); - } - if (nodeStats.http() != null) { - nodeStats.http().toXContent(builder, request); - } - - builder.endObject(); - } - builder.endObject(); - + response.toXContent(builder, request); builder.endObject(); channel.sendResponse(new XContentRestResponse(request, RestStatus.OK, builder)); } catch (Exception e) { @@ -108,4 +112,67 @@ public class RestNodesStatsAction extends BaseRestHandler { } }); } + + class RestIndicesHandler implements RestHandler { + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(RestActions.splitNodes(request.param("nodeId"))); + nodesStatsRequest.clear().indices(true); + executeNodeStats(request, channel, nodesStatsRequest); + } + } + + class RestOsHandler implements RestHandler { + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(RestActions.splitNodes(request.param("nodeId"))); + nodesStatsRequest.clear().os(true); + executeNodeStats(request, channel, nodesStatsRequest); + } + } + + class RestProcessHandler implements RestHandler { + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(RestActions.splitNodes(request.param("nodeId"))); + nodesStatsRequest.clear().process(true); + executeNodeStats(request, channel, nodesStatsRequest); + } + } + + class RestJvmHandler implements RestHandler { + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(RestActions.splitNodes(request.param("nodeId"))); + nodesStatsRequest.clear().jvm(true); + executeNodeStats(request, channel, nodesStatsRequest); + } + } + + class RestNetworkHandler implements RestHandler { + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(RestActions.splitNodes(request.param("nodeId"))); + nodesStatsRequest.clear().network(true); + executeNodeStats(request, channel, nodesStatsRequest); + } + } + + class RestTransportHandler implements RestHandler { + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(RestActions.splitNodes(request.param("nodeId"))); + nodesStatsRequest.clear().transport(true); + executeNodeStats(request, channel, nodesStatsRequest); + } + } + + class RestHttpHandler implements RestHandler { + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(RestActions.splitNodes(request.param("nodeId"))); + nodesStatsRequest.clear().http(true); + executeNodeStats(request, channel, nodesStatsRequest); + } + } } \ No newline at end of file From 5b2854e8bb8ecbd240429a751bf3b95b86146c17 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 9 Jan 2012 21:28:56 +0200 Subject: [PATCH 148/270] Date Histogram Facet: Add `pre_offset` and `post_offset` options, closes #1599. --- .../common/joda/TimeZoneRounding.java | 39 ++++++++++++++++++- .../DateHistogramFacetProcessor.java | 19 ++++++++- .../unit/deps/joda/TimeZoneRoundingTests.java | 6 +++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/common/joda/TimeZoneRounding.java b/src/main/java/org/elasticsearch/common/joda/TimeZoneRounding.java index ac4cbe3d2f0..feb9d8fdf21 100644 --- a/src/main/java/org/elasticsearch/common/joda/TimeZoneRounding.java +++ b/src/main/java/org/elasticsearch/common/joda/TimeZoneRounding.java @@ -48,6 +48,9 @@ public abstract class TimeZoneRounding { private float factor = 1.0f; + private long preOffset; + private long postOffset; + public Builder(DateTimeField field) { this.field = field; this.interval = -1; @@ -68,6 +71,16 @@ public abstract class TimeZoneRounding { return this; } + public Builder preOffset(long preOffset) { + this.preOffset = preOffset; + return this; + } + + public Builder postOffset(long postOffset) { + this.postOffset = postOffset; + return this; + } + public Builder factor(float factor) { this.factor = factor; return this; @@ -77,7 +90,7 @@ public abstract class TimeZoneRounding { TimeZoneRounding timeZoneRounding; if (field != null) { if (preTz.equals(DateTimeZone.UTC) && postTz.equals(DateTimeZone.UTC)) { - return new UTCTimeZoneRoundingFloor(field); + timeZoneRounding = new UTCTimeZoneRoundingFloor(field); } else if (field.getDurationField().getUnitMillis() < DateTimeConstants.MILLIS_PER_HOUR * 12) { timeZoneRounding = new TimeTimeZoneRoundingFloor(field, preTz, postTz); } else { @@ -85,13 +98,16 @@ public abstract class TimeZoneRounding { } } else { if (preTz.equals(DateTimeZone.UTC) && postTz.equals(DateTimeZone.UTC)) { - return new UTCIntervalTimeZoneRounding(interval); + timeZoneRounding = new UTCIntervalTimeZoneRounding(interval); } else if (interval < DateTimeConstants.MILLIS_PER_HOUR * 12) { timeZoneRounding = new TimeIntervalTimeZoneRounding(interval, preTz, postTz); } else { timeZoneRounding = new DayIntervalTimeZoneRounding(interval, preTz, postTz); } } + if (preOffset != 0 || postOffset != 0) { + timeZoneRounding = new PrePostTimeZoneRounding(timeZoneRounding, preOffset, postOffset); + } if (factor != 1.0f) { timeZoneRounding = new FactorTimeZoneRounding(timeZoneRounding, factor); } @@ -237,4 +253,23 @@ public abstract class TimeZoneRounding { return timeZoneRounding.calc((long) (factor * utcMillis)); } } + + static class PrePostTimeZoneRounding extends TimeZoneRounding { + + private final TimeZoneRounding timeZoneRounding; + + private final long preOffset; + private final long postOffset; + + PrePostTimeZoneRounding(TimeZoneRounding timeZoneRounding, long preOffset, long postOffset) { + this.timeZoneRounding = timeZoneRounding; + this.preOffset = preOffset; + this.postOffset = postOffset; + } + + @Override + public long calc(long utcMillis) { + return postOffset + timeZoneRounding.calc(utcMillis + preOffset); + } + } } diff --git a/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java b/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java index 54199862da0..d656debc744 100644 --- a/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java +++ b/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java @@ -88,6 +88,8 @@ public class DateHistogramFacetProcessor extends AbstractComponent implements Fa String interval = null; DateTimeZone preZone = DateTimeZone.UTC; DateTimeZone postZone = DateTimeZone.UTC; + long preOffset = 0; + long postOffset = 0; float factor = 1.0f; Chronology chronology = ISOChronology.getInstanceUTC(); DateHistogramFacet.ComparatorType comparatorType = DateHistogramFacet.ComparatorType.TIME; @@ -115,6 +117,10 @@ public class DateHistogramFacetProcessor extends AbstractComponent implements Fa preZone = parseZone(parser, token); } else if ("post_zone".equals(fieldName) || "postZone".equals(fieldName)) { postZone = parseZone(parser, token); + } else if ("pre_offset".equals(fieldName) || "preOffset".equals(fieldName)) { + preOffset = parseOffset(parser.text()); + } else if ("post_offset".equals(fieldName) || "postOffset".equals(fieldName)) { + postOffset = parseOffset(parser.text()); } else if ("factor".equals(fieldName)) { factor = parser.floatValue(); } else if ("value_script".equals(fieldName) || "valueScript".equals(fieldName)) { @@ -152,7 +158,11 @@ public class DateHistogramFacetProcessor extends AbstractComponent implements Fa tzRoundingBuilder = TimeZoneRounding.builder(TimeValue.parseTimeValue(interval, null)); } - TimeZoneRounding tzRounding = tzRoundingBuilder.preZone(preZone).postZone(postZone).factor(factor).build(); + TimeZoneRounding tzRounding = tzRoundingBuilder + .preZone(preZone).postZone(postZone) + .preOffset(preOffset).postOffset(postOffset) + .factor(factor) + .build(); if (valueScript != null) { return new ValueScriptDateHistogramFacetCollector(facetName, keyField, scriptLang, valueScript, params, tzRounding, comparatorType, context); @@ -163,6 +173,13 @@ public class DateHistogramFacetProcessor extends AbstractComponent implements Fa } } + private long parseOffset(String offset) throws IOException { + if (offset.charAt(0) == '-') { + return -TimeValue.parseTimeValue(offset.substring(1), null).millis(); + } + return TimeValue.parseTimeValue(offset, null).millis(); + } + private DateTimeZone parseZone(XContentParser parser, XContentParser.Token token) throws IOException { if (token == XContentParser.Token.VALUE_NUMBER) { return DateTimeZone.forOffsetHours(parser.intValue()); diff --git a/src/test/java/org/elasticsearch/test/unit/deps/joda/TimeZoneRoundingTests.java b/src/test/java/org/elasticsearch/test/unit/deps/joda/TimeZoneRoundingTests.java index 9d59516f1b5..524c488b9da 100644 --- a/src/test/java/org/elasticsearch/test/unit/deps/joda/TimeZoneRoundingTests.java +++ b/src/test/java/org/elasticsearch/test/unit/deps/joda/TimeZoneRoundingTests.java @@ -39,6 +39,12 @@ public class TimeZoneRoundingTests { public void testUTCMonthRounding() { TimeZoneRounding tzRounding = TimeZoneRounding.builder(chronology().monthOfYear()).build(); assertThat(tzRounding.calc(utc("2009-02-03T01:01:01")), equalTo(utc("2009-02-01T00:00:00.000Z"))); + + tzRounding = TimeZoneRounding.builder(chronology().weekOfWeekyear()).build(); + assertThat(tzRounding.calc(utc("2012-01-10T01:01:01")), equalTo(utc("2012-01-09T00:00:00.000Z"))); + + tzRounding = TimeZoneRounding.builder(chronology().weekOfWeekyear()).postOffset(-TimeValue.timeValueHours(24).millis()).build(); + assertThat(tzRounding.calc(utc("2012-01-10T01:01:01")), equalTo(utc("2012-01-08T00:00:00.000Z"))); } @Test From 710e0d78df259dd3cfb75d18f2c8d4eae71b804c Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 9 Jan 2012 23:07:38 +0200 Subject: [PATCH 149/270] support Iterable for generic iterable based structures for builder --- .../common/xcontent/XContentBuilder.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index 1ad802fc77d..cf3d32a706c 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -34,7 +34,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.Date; -import java.util.List; import java.util.Map; /** @@ -486,7 +485,7 @@ public final class XContentBuilder { return this; } - public XContentBuilder field(String name, List value) throws IOException { + public XContentBuilder field(String name, Iterable value) throws IOException { startArray(name); for (Object o : value) { value(o); @@ -495,7 +494,7 @@ public final class XContentBuilder { return this; } - public XContentBuilder field(XContentBuilderString name, List value) throws IOException { + public XContentBuilder field(XContentBuilderString name, Iterable value) throws IOException { startArray(name); for (Object o : value) { value(o); @@ -644,8 +643,8 @@ public final class XContentBuilder { } else if (value instanceof Map) { //noinspection unchecked field(name, (Map) value); - } else if (value instanceof List) { - field(name, (List) value); + } else if (value instanceof Iterable) { + field(name, (Iterable) value); } else if (value instanceof Object[]) { field(name, (Object[]) value); } else if (value instanceof int[]) { @@ -695,8 +694,8 @@ public final class XContentBuilder { } else if (value instanceof Map) { //noinspection unchecked field(name, (Map) value); - } else if (value instanceof List) { - field(name, (List) value); + } else if (value instanceof Iterable) { + field(name, (Iterable) value); } else if (value instanceof Object[]) { field(name, (Object[]) value); } else if (value instanceof int[]) { From ed8a46ce09ccf43adec55af91a72fc9c5dc1ec05 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 10 Jan 2012 10:43:01 +0200 Subject: [PATCH 150/270] on index failure when updating, make sure to call the listener --- .../org/elasticsearch/action/update/TransportUpdateAction.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java index 888e1aafe65..5a296cdf925 100644 --- a/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java +++ b/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -220,6 +220,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio return; } } + listener.onFailure(e); } }); } else if ("delete".equals(operation)) { From 682176497f9f454943d2373dc24501ad37b8dbb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20Deve=CC=80ze?= Date: Tue, 10 Jan 2012 00:22:18 +0100 Subject: [PATCH 151/270] handle timestamp and TTL in update action --- .../action/update/TransportUpdateAction.java | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java index 5a296cdf925..ae9eea79b9f 100644 --- a/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java +++ b/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -43,6 +43,7 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.engine.DocumentMissingException; @@ -52,7 +53,7 @@ import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.internal.ParentFieldMapper; import org.elasticsearch.index.mapper.internal.RoutingFieldMapper; import org.elasticsearch.index.mapper.internal.SourceFieldMapper; -import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; +import org.elasticsearch.index.mapper.internal.TTLFieldMapper; import org.elasticsearch.index.service.IndexService; import org.elasticsearch.index.shard.IllegalIndexShardStateException; import org.elasticsearch.index.shard.ShardId; @@ -152,8 +153,9 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio IndexService indexService = indicesService.indexServiceSafe(request.index()); IndexShard indexShard = indexService.shardSafe(request.shardId()); + long getDate = System.currentTimeMillis(); GetResult getResult = indexShard.getService().get(request.type(), request.id(), - new String[]{SourceFieldMapper.NAME, RoutingFieldMapper.NAME, ParentFieldMapper.NAME, TimestampFieldMapper.NAME}, true); + new String[]{SourceFieldMapper.NAME, RoutingFieldMapper.NAME, ParentFieldMapper.NAME, TTLFieldMapper.NAME}, true); // no doc, what to do, what to do... if (!getResult.exists()) { @@ -183,12 +185,28 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio } String operation = (String) ctx.get("op"); + String timestamp = (String) ctx.get("_timestamp"); + Long ttl = null; + Object fetchedTTL = ctx.get("_ttl"); + if (fetchedTTL != null) { + if (fetchedTTL instanceof Number) { + ttl = ((Number) fetchedTTL).longValue(); + } else { + ttl = TimeValue.parseTimeValue((String) fetchedTTL, null).millis(); + } + } source = (Map) ctx.get("_source"); // apply script to update the source String routing = getResult.fields().containsKey(RoutingFieldMapper.NAME) ? getResult.field(RoutingFieldMapper.NAME).value().toString() : null; String parent = getResult.fields().containsKey(ParentFieldMapper.NAME) ? getResult.field(ParentFieldMapper.NAME).value().toString() : null; - // TODO ttl/timestamp + // No TTL has been given in the update script so we keep previous TTL value if there is one + if (ttl == null) { + ttl = getResult.fields().containsKey(TTLFieldMapper.NAME) ? (Long) getResult.field(TTLFieldMapper.NAME).value() : null; + if (ttl != null) { + ttl = ttl - (System.currentTimeMillis() - getDate); // It is an approximation of exact TTL value, could be improved + } + } // TODO percolate? @@ -197,7 +215,8 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio if (operation == null || "index".equals(operation)) { IndexRequest indexRequest = Requests.indexRequest(request.index()).type(request.type()).id(request.id()).routing(routing).parent(parent) .source(source, sourceAndContent.v1()) - .version(getResult.version()).replicationType(request.replicationType()).consistencyLevel(request.consistencyLevel()); + .version(getResult.version()).replicationType(request.replicationType()).consistencyLevel(request.consistencyLevel()) + .timestamp(timestamp).ttl(ttl); indexRequest.operationThreaded(false); indexAction.execute(indexRequest, new ActionListener() { @Override From ecd41ca6e4eaf84e1f6bb2ae185f367b68396cdc Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 10 Jan 2012 13:10:53 +0200 Subject: [PATCH 152/270] add 0.18.8 --- src/main/java/org/elasticsearch/Version.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/elasticsearch/Version.java b/src/main/java/org/elasticsearch/Version.java index ba75c07541b..1672a4b8ce4 100644 --- a/src/main/java/org/elasticsearch/Version.java +++ b/src/main/java/org/elasticsearch/Version.java @@ -50,6 +50,8 @@ public class Version { public static final Version V_0_18_6 = new Version(V_0_18_6_ID, false); public static final int V_0_18_7_ID = /*00*/180799; public static final Version V_0_18_7 = new Version(V_0_18_7_ID, false); + public static final int V_0_18_8_ID = /*00*/180899; + public static final Version V_0_18_8 = new Version(V_0_18_8_ID, false); public static final int V_0_19_0_ID = /*00*/190099; public static final Version V_0_19_0 = new Version(V_0_19_0_ID, true); @@ -78,6 +80,8 @@ public class Version { return V_0_18_6; case V_0_18_7_ID: return V_0_18_7; + case V_0_18_8_ID: + return V_0_18_8; case V_0_19_0_ID: return V_0_19_0; From 3b92962ddd7c24b6fda871251b522c9c023f2889 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 10 Jan 2012 14:17:26 +0200 Subject: [PATCH 153/270] bring back the option to use max instead of size for min/max thread pools --- src/main/java/org/elasticsearch/threadpool/ThreadPool.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index e0cd7ae3c34..0caff8b7522 100644 --- a/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -191,13 +191,13 @@ public class ThreadPool extends AbstractComponent { } else if ("scaling".equals(type)) { TimeValue keepAlive = settings.getAsTime("keep_alive", defaultSettings.getAsTime("keep_alive", timeValueMinutes(5))); int min = settings.getAsInt("min", defaultSettings.getAsInt("min", 1)); - int size = settings.getAsInt("size", defaultSettings.getAsInt("size", Runtime.getRuntime().availableProcessors() * 5)); + int size = settings.getAsInt("max", settings.getAsInt("size", defaultSettings.getAsInt("size", Runtime.getRuntime().availableProcessors() * 5))); logger.debug("creating thread_pool [{}], type [{}], min [{}], size [{}], keep_alive [{}]", name, type, min, size, keepAlive); return EsExecutors.newScalingExecutorService(min, size, keepAlive.millis(), TimeUnit.MILLISECONDS, threadFactory); } else if ("blocking".equals(type)) { TimeValue keepAlive = settings.getAsTime("keep_alive", defaultSettings.getAsTime("keep_alive", timeValueMinutes(5))); int min = settings.getAsInt("min", defaultSettings.getAsInt("min", 1)); - int size = settings.getAsInt("size", defaultSettings.getAsInt("size", Runtime.getRuntime().availableProcessors() * 5)); + int size = settings.getAsInt("max", settings.getAsInt("size", defaultSettings.getAsInt("size", Runtime.getRuntime().availableProcessors() * 5))); SizeValue capacity = settings.getAsSize("queue_size", defaultSettings.getAsSize("queue_size", new SizeValue(1000))); TimeValue waitTime = settings.getAsTime("wait_time", defaultSettings.getAsTime("wait_time", timeValueSeconds(60))); logger.debug("creating thread_pool [{}], type [{}], min [{}], size [{}], queue_size [{}], keep_alive [{}], wait_time [{}]", name, type, min, size, capacity.singles(), keepAlive, waitTime); From 9d979dfc015ee8de43a1080d817f48a1583b8ae9 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 10 Jan 2012 17:45:10 +0200 Subject: [PATCH 154/270] Add thread_pool to nodes info and nodes stats APIs, closes #1601. --- .../admin/cluster/node/info/NodeInfo.java | 26 +- .../cluster/node/info/NodesInfoRequest.java | 19 ++ .../cluster/node/info/NodesInfoResponse.java | 3 + .../node/info/TransportNodesInfoAction.java | 2 +- .../admin/cluster/node/stats/NodeStats.java | 32 ++- .../cluster/node/stats/NodesStatsRequest.java | 19 ++ .../node/stats/NodesStatsResponse.java | 3 + .../node/stats/TransportNodesStatsAction.java | 2 +- .../node/info/NodesInfoRequestBuilder.java | 8 + .../node/stats/NodesStatsRequestBuilder.java | 8 + .../node/service/NodeService.java | 43 +++- .../node/info/RestNodesInfoAction.java | 13 + .../node/stats/RestNodesStatsAction.java | 13 + .../elasticsearch/threadpool/ThreadPool.java | 233 ++++++++++++++++-- .../threadpool/ThreadPoolInfo.java | 86 +++++++ .../threadpool/ThreadPoolStats.java | 154 ++++++++++++ 16 files changed, 625 insertions(+), 39 deletions(-) create mode 100644 src/main/java/org/elasticsearch/threadpool/ThreadPoolInfo.java create mode 100644 src/main/java/org/elasticsearch/threadpool/ThreadPoolStats.java diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java index f282d2d47d1..b2e81f26950 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java @@ -32,6 +32,7 @@ import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.monitor.network.NetworkInfo; import org.elasticsearch.monitor.os.OsInfo; import org.elasticsearch.monitor.process.ProcessInfo; +import org.elasticsearch.threadpool.ThreadPoolInfo; import org.elasticsearch.transport.TransportInfo; import java.io.IOException; @@ -60,6 +61,9 @@ public class NodeInfo extends NodeOperationResponse { @Nullable private JvmInfo jvm; + @Nullable + private ThreadPoolInfo threadPool; + @Nullable private NetworkInfo network; @@ -73,7 +77,7 @@ public class NodeInfo extends NodeOperationResponse { } public NodeInfo(@Nullable String hostname, DiscoveryNode node, @Nullable ImmutableMap serviceAttributes, @Nullable Settings settings, - @Nullable OsInfo os, @Nullable ProcessInfo process, @Nullable JvmInfo jvm, @Nullable NetworkInfo network, + @Nullable OsInfo os, @Nullable ProcessInfo process, @Nullable JvmInfo jvm, @Nullable ThreadPoolInfo threadPool, @Nullable NetworkInfo network, @Nullable TransportInfo transport, @Nullable HttpInfo http) { super(node); this.hostname = hostname; @@ -82,6 +86,7 @@ public class NodeInfo extends NodeOperationResponse { this.os = os; this.process = process; this.jvm = jvm; + this.threadPool = threadPool; this.network = network; this.transport = transport; this.http = http; @@ -183,6 +188,16 @@ public class NodeInfo extends NodeOperationResponse { return jvm(); } + @Nullable + public ThreadPoolInfo threadPool() { + return this.threadPool; + } + + @Nullable + public ThreadPoolInfo getThreadPool() { + return threadPool(); + } + /** * Network level information. */ @@ -251,6 +266,9 @@ public class NodeInfo extends NodeOperationResponse { if (in.readBoolean()) { jvm = JvmInfo.readJvmInfo(in); } + if (in.readBoolean()) { + threadPool = ThreadPoolInfo.readThreadPoolInfo(in); + } if (in.readBoolean()) { network = NetworkInfo.readNetworkInfo(in); } @@ -305,6 +323,12 @@ public class NodeInfo extends NodeOperationResponse { out.writeBoolean(true); jvm.writeTo(out); } + if (threadPool == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + threadPool.writeTo(out); + } if (network == null) { out.writeBoolean(false); } else { diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java index fdd033ace03..5c882e8743e 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java @@ -34,6 +34,7 @@ public class NodesInfoRequest extends NodesOperationRequest { private boolean os = false; private boolean process = false; private boolean jvm = false; + private boolean threadPool = false; private boolean network = false; private boolean transport = false; private boolean http = false; @@ -57,6 +58,7 @@ public class NodesInfoRequest extends NodesOperationRequest { os = false; process = false; jvm = false; + threadPool = false; network = false; transport = false; http = false; @@ -123,6 +125,21 @@ public class NodesInfoRequest extends NodesOperationRequest { return this; } + /** + * Should the node Thread Pool info be returned. + */ + public boolean threadPool() { + return this.threadPool; + } + + /** + * Should the node Thread Pool info be returned. + */ + public NodesInfoRequest threadPool(boolean threadPool) { + this.threadPool = threadPool; + return this; + } + /** * Should the node Network be returned. */ @@ -175,6 +192,7 @@ public class NodesInfoRequest extends NodesOperationRequest { os = in.readBoolean(); process = in.readBoolean(); jvm = in.readBoolean(); + threadPool = in.readBoolean(); network = in.readBoolean(); transport = in.readBoolean(); http = in.readBoolean(); @@ -187,6 +205,7 @@ public class NodesInfoRequest extends NodesOperationRequest { out.writeBoolean(os); out.writeBoolean(process); out.writeBoolean(jvm); + out.writeBoolean(threadPool); out.writeBoolean(network); out.writeBoolean(transport); out.writeBoolean(http); diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java index 8b6fc920eb0..86d3d42a5e5 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java @@ -116,6 +116,9 @@ public class NodesInfoResponse extends NodesOperationResponse implemen if (nodeInfo.jvm() != null) { nodeInfo.jvm().toXContent(builder, params); } + if (nodeInfo.threadPool() != null) { + nodeInfo.threadPool().toXContent(builder, params); + } if (nodeInfo.network() != null) { nodeInfo.network().toXContent(builder, params); } diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java index ddca44262d5..b5f6d8a1341 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java @@ -98,7 +98,7 @@ public class TransportNodesInfoAction extends TransportNodesOperationAction implem if (nodeStats.jvm() != null) { nodeStats.jvm().toXContent(builder, params); } + if (nodeStats.threadPool() != null) { + nodeStats.threadPool().toXContent(builder, params); + } if (nodeStats.network() != null) { nodeStats.network().toXContent(builder, params); } diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index 3ccb8a5dc65..e93f595544f 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -98,7 +98,7 @@ public class TransportNodesStatsAction extends TransportNodesOperationAction executors; + private final ImmutableMap executors; private final ScheduledThreadPoolExecutor scheduler; @@ -73,7 +81,7 @@ public class ThreadPool extends AbstractComponent { Map groupSettings = settings.getGroups("threadpool"); - Map executors = Maps.newHashMap(); + Map executors = Maps.newHashMap(); executors.put(Names.CACHED, build(Names.CACHED, "cached", groupSettings.get(Names.CACHED), settingsBuilder().put("keep_alive", "30s").build())); executors.put(Names.INDEX, build(Names.INDEX, "cached", groupSettings.get(Names.INDEX), ImmutableSettings.Builder.EMPTY_SETTINGS)); executors.put(Names.SEARCH, build(Names.SEARCH, "cached", groupSettings.get(Names.SEARCH), ImmutableSettings.Builder.EMPTY_SETTINGS)); @@ -81,7 +89,7 @@ public class ThreadPool extends AbstractComponent { executors.put(Names.MANAGEMENT, build(Names.MANAGEMENT, "scaling", groupSettings.get(Names.MANAGEMENT), settingsBuilder().put("keep_alive", "5m").put("size", 20).build())); executors.put(Names.MERGE, build(Names.MERGE, "scaling", groupSettings.get(Names.MERGE), settingsBuilder().put("keep_alive", "5m").put("size", 20).build())); executors.put(Names.SNAPSHOT, build(Names.SNAPSHOT, "scaling", groupSettings.get(Names.SNAPSHOT), ImmutableSettings.Builder.EMPTY_SETTINGS)); - executors.put(Names.SAME, MoreExecutors.sameThreadExecutor()); + executors.put(Names.SAME, new ExecutorHolder(MoreExecutors.sameThreadExecutor(), new Info(Names.SAME, "same"))); this.executors = ImmutableMap.copyOf(executors); this.scheduler = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1, EsExecutors.daemonThreadFactory(settings, "[scheduler]")); this.scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); @@ -96,12 +104,40 @@ public class ThreadPool extends AbstractComponent { return estimatedTimeThread.estimatedTimeInMillis(); } + public ThreadPoolInfo info() { + List infos = new ArrayList(); + for (ExecutorHolder holder : executors.values()) { + String name = holder.info.name(); + // no need to have info on "same" thread pool + if ("same".equals(name)) { + continue; + } + infos.add(holder.info); + } + return new ThreadPoolInfo(infos); + } + + public ThreadPoolStats stats() { + List stats = new ArrayList(); + for (ExecutorHolder holder : executors.values()) { + String name = holder.info.name(); + // no need to have info on "same" thread pool + if ("same".equals(name)) { + continue; + } + int threads = ((ThreadPoolExecutor) holder.executor).getPoolSize(); + int queue = ((ThreadPoolExecutor) holder.executor).getQueue().size(); + stats.add(new ThreadPoolStats.Stats(name, threads, queue)); + } + return new ThreadPoolStats(stats); + } + public Executor cached() { return executor(Names.CACHED); } public Executor executor(String name) { - Executor executor = executors.get(name); + Executor executor = executors.get(name).executor; if (executor == null) { throw new ElasticSearchIllegalArgumentException("No executor found for [" + name + "]"); } @@ -127,9 +163,9 @@ public class ThreadPool extends AbstractComponent { estimatedTimeThread.running = false; estimatedTimeThread.interrupt(); scheduler.shutdown(); - for (Executor executor : executors.values()) { - if (executor instanceof ThreadPoolExecutor) { - ((ThreadPoolExecutor) executor).shutdown(); + for (ExecutorHolder executor : executors.values()) { + if (executor.executor instanceof ThreadPoolExecutor) { + ((ThreadPoolExecutor) executor.executor).shutdown(); } } } @@ -138,24 +174,24 @@ public class ThreadPool extends AbstractComponent { estimatedTimeThread.running = false; estimatedTimeThread.interrupt(); scheduler.shutdownNow(); - for (Executor executor : executors.values()) { - if (executor instanceof ThreadPoolExecutor) { - ((ThreadPoolExecutor) executor).shutdownNow(); + for (ExecutorHolder executor : executors.values()) { + if (executor.executor instanceof ThreadPoolExecutor) { + ((ThreadPoolExecutor) executor.executor).shutdownNow(); } } } public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { boolean result = scheduler.awaitTermination(timeout, unit); - for (Executor executor : executors.values()) { - if (executor instanceof ThreadPoolExecutor) { - result &= ((ThreadPoolExecutor) executor).awaitTermination(timeout, unit); + for (ExecutorHolder executor : executors.values()) { + if (executor.executor instanceof ThreadPoolExecutor) { + result &= ((ThreadPoolExecutor) executor.executor).awaitTermination(timeout, unit); } } return result; } - private Executor build(String name, String defaultType, @Nullable Settings settings, Settings defaultSettings) { + private ExecutorHolder build(String name, String defaultType, @Nullable Settings settings, Settings defaultSettings) { if (settings == null) { settings = ImmutableSettings.Builder.EMPTY_SETTINGS; } @@ -163,17 +199,18 @@ public class ThreadPool extends AbstractComponent { ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(settings, "[" + name + "]"); if ("same".equals(type)) { logger.debug("creating thread_pool [{}], type [{}]", name, type); - return MoreExecutors.sameThreadExecutor(); + return new ExecutorHolder(MoreExecutors.sameThreadExecutor(), new Info(name, type)); } else if ("cached".equals(type)) { TimeValue keepAlive = settings.getAsTime("keep_alive", defaultSettings.getAsTime("keep_alive", timeValueMinutes(5))); logger.debug("creating thread_pool [{}], type [{}], keep_alive [{}]", name, type, keepAlive); - return new ThreadPoolExecutor(0, Integer.MAX_VALUE, + Executor executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, keepAlive.millis(), TimeUnit.MILLISECONDS, new SynchronousQueue(), threadFactory); + return new ExecutorHolder(executor, new Info(name, type, -1, -1, keepAlive, null)); } else if ("fixed".equals(type)) { int size = settings.getAsInt("size", defaultSettings.getAsInt("size", Runtime.getRuntime().availableProcessors() * 5)); - int queueSize = settings.getAsInt("queue_size", defaultSettings.getAsInt("queue_size", -1)); + SizeValue capacity = settings.getAsSize("capacity", settings.getAsSize("queue_size", defaultSettings.getAsSize("queue_size", null))); RejectedExecutionHandler rejectedExecutionHandler; String rejectSetting = settings.get("reject_policy", defaultSettings.get("reject_policy", "abort")); if ("abort".equals(rejectSetting)) { @@ -183,25 +220,28 @@ public class ThreadPool extends AbstractComponent { } else { throw new ElasticSearchIllegalArgumentException("reject_policy [" + rejectSetting + "] not valid for [" + name + "] thread pool"); } - logger.debug("creating thread_pool [{}], type [{}], size [{}], queue_size [{}], reject_policy [{}]", name, type, size, queueSize, rejectSetting); - return new ThreadPoolExecutor(size, size, + logger.debug("creating thread_pool [{}], type [{}], size [{}], queue_size [{}], reject_policy [{}]", name, type, size, capacity, rejectSetting); + Executor executor = new ThreadPoolExecutor(size, size, 0L, TimeUnit.MILLISECONDS, - queueSize <= 0 ? new LinkedTransferQueue() : new ArrayBlockingQueue(queueSize), + capacity == null ? new LinkedTransferQueue() : new ArrayBlockingQueue((int) capacity.singles()), threadFactory, rejectedExecutionHandler); + return new ExecutorHolder(executor, new Info(name, type, size, size, null, capacity)); } else if ("scaling".equals(type)) { TimeValue keepAlive = settings.getAsTime("keep_alive", defaultSettings.getAsTime("keep_alive", timeValueMinutes(5))); int min = settings.getAsInt("min", defaultSettings.getAsInt("min", 1)); int size = settings.getAsInt("max", settings.getAsInt("size", defaultSettings.getAsInt("size", Runtime.getRuntime().availableProcessors() * 5))); logger.debug("creating thread_pool [{}], type [{}], min [{}], size [{}], keep_alive [{}]", name, type, min, size, keepAlive); - return EsExecutors.newScalingExecutorService(min, size, keepAlive.millis(), TimeUnit.MILLISECONDS, threadFactory); + Executor executor = EsExecutors.newScalingExecutorService(min, size, keepAlive.millis(), TimeUnit.MILLISECONDS, threadFactory); + return new ExecutorHolder(executor, new Info(name, type, min, size, keepAlive, null)); } else if ("blocking".equals(type)) { TimeValue keepAlive = settings.getAsTime("keep_alive", defaultSettings.getAsTime("keep_alive", timeValueMinutes(5))); int min = settings.getAsInt("min", defaultSettings.getAsInt("min", 1)); int size = settings.getAsInt("max", settings.getAsInt("size", defaultSettings.getAsInt("size", Runtime.getRuntime().availableProcessors() * 5))); - SizeValue capacity = settings.getAsSize("queue_size", defaultSettings.getAsSize("queue_size", new SizeValue(1000))); + SizeValue capacity = settings.getAsSize("capacity", settings.getAsSize("queue_size", defaultSettings.getAsSize("queue_size", new SizeValue(1000)))); TimeValue waitTime = settings.getAsTime("wait_time", defaultSettings.getAsTime("wait_time", timeValueSeconds(60))); logger.debug("creating thread_pool [{}], type [{}], min [{}], size [{}], queue_size [{}], keep_alive [{}], wait_time [{}]", name, type, min, size, capacity.singles(), keepAlive, waitTime); - return EsExecutors.newBlockingExecutorService(min, size, keepAlive.millis(), TimeUnit.MILLISECONDS, threadFactory, (int) capacity.singles(), waitTime.millis(), TimeUnit.MILLISECONDS); + Executor executor = EsExecutors.newBlockingExecutorService(min, size, keepAlive.millis(), TimeUnit.MILLISECONDS, threadFactory, (int) capacity.singles(), waitTime.millis(), TimeUnit.MILLISECONDS); + return new ExecutorHolder(executor, new Info(name, type, min, size, keepAlive, capacity)); } throw new ElasticSearchIllegalArgumentException("No type found [" + type + "], for [" + name + "]"); } @@ -307,4 +347,151 @@ public class ThreadPool extends AbstractComponent { } } } + + static class ExecutorHolder { + public final Executor executor; + public final Info info; + + ExecutorHolder(Executor executor, Info info) { + this.executor = executor; + this.info = info; + } + } + + public static class Info implements Streamable, ToXContent { + + private String name; + private String type; + private int min; + private int max; + private TimeValue keepAlive; + private SizeValue capacity; + + Info() { + + } + + public Info(String name, String type) { + this(name, type, -1); + } + + public Info(String name, String type, int size) { + this(name, type, size, size, null, null); + } + + public Info(String name, String type, int min, int max, @Nullable TimeValue keepAlive, @Nullable SizeValue capacity) { + this.name = name; + this.type = type; + this.min = min; + this.max = max; + this.keepAlive = keepAlive; + this.capacity = capacity; + } + + public String name() { + return this.name; + } + + public String getName() { + return this.name; + } + + public String type() { + return this.type; + } + + public String getType() { + return this.type; + } + + public int min() { + return this.min; + } + + public int getMin() { + return this.min; + } + + public int max() { + return this.max; + } + + public int getMax() { + return this.max; + } + + @Nullable + public TimeValue keepAlive() { + return this.keepAlive; + } + + @Nullable + public TimeValue getKeepAlive() { + return this.keepAlive; + } + + @Nullable + public SizeValue capacity() { + return this.capacity; + } + + @Nullable + public SizeValue getCapacity() { + return this.capacity; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + name = in.readUTF(); + type = in.readUTF(); + min = in.readInt(); + max = in.readInt(); + if (in.readBoolean()) { + keepAlive = TimeValue.readTimeValue(in); + } + if (in.readBoolean()) { + capacity = SizeValue.readSizeValue(in); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeUTF(name); + out.writeUTF(type); + out.writeInt(min); + out.writeInt(max); + if (keepAlive == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + keepAlive.writeTo(out); + } + if (capacity == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + capacity.writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(name, XContentBuilder.FieldCaseConversion.NONE); + builder.field("type", type); + if (min != -1) { + builder.field("min", min); + } + if (max != -1) { + builder.field("max", max); + } + if (keepAlive != null) { + builder.field("keep_alive", keepAlive.toString()); + } + if (capacity != null) { + builder.field("capacity", capacity.toString()); + } + builder.endObject(); + return builder; + } + } } diff --git a/src/main/java/org/elasticsearch/threadpool/ThreadPoolInfo.java b/src/main/java/org/elasticsearch/threadpool/ThreadPoolInfo.java new file mode 100644 index 00000000000..dda8b25d4a0 --- /dev/null +++ b/src/main/java/org/elasticsearch/threadpool/ThreadPoolInfo.java @@ -0,0 +1,86 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.threadpool; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + */ +public class ThreadPoolInfo implements Streamable, Iterable, ToXContent { + + private List infos; + + ThreadPoolInfo() { + } + + + public ThreadPoolInfo(List infos) { + this.infos = infos; + } + + @Override + public Iterator iterator() { + return infos.iterator(); + } + + public static ThreadPoolInfo readThreadPoolInfo(StreamInput in) throws IOException { + ThreadPoolInfo info = new ThreadPoolInfo(); + info.readFrom(in); + return info; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + int size = in.readVInt(); + infos = new ArrayList(size); + for (int i = 0; i < size; i++) { + ThreadPool.Info info = new ThreadPool.Info(); + info.readFrom(in); + infos.add(info); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(infos.size()); + for (ThreadPool.Info info : infos) { + info.writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("thread_pool"); + for (ThreadPool.Info info : infos) { + info.toXContent(builder, params); + } + builder.endObject(); + return builder; + } +} diff --git a/src/main/java/org/elasticsearch/threadpool/ThreadPoolStats.java b/src/main/java/org/elasticsearch/threadpool/ThreadPoolStats.java new file mode 100644 index 00000000000..a9476cfd1cd --- /dev/null +++ b/src/main/java/org/elasticsearch/threadpool/ThreadPoolStats.java @@ -0,0 +1,154 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.threadpool; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + */ +public class ThreadPoolStats implements Streamable, ToXContent, Iterable { + + public static class Stats implements Streamable, ToXContent { + + private String name; + private int threads; + private int queue; + + Stats() { + + } + + public Stats(String name, int threads, int queue) { + this.name = name; + this.threads = threads; + this.queue = queue; + } + + public String name() { + return this.name; + } + + public String getName() { + return this.name; + } + + public int threads() { + return this.threads; + } + + public int getThreads() { + return this.threads; + } + + public int queue() { + return this.queue; + } + + public int getQueue() { + return this.queue; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + name = in.readUTF(); + threads = in.readInt(); + queue = in.readInt(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeUTF(name); + out.writeInt(threads); + out.writeInt(queue); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(name, XContentBuilder.FieldCaseConversion.NONE); + if (threads != -1) { + builder.field("threads", threads); + } + if (queue != -1) { + builder.field("queue", queue); + } + builder.endObject(); + return builder; + } + } + + private List stats; + + ThreadPoolStats() { + + } + + public ThreadPoolStats(List stats) { + this.stats = stats; + } + + @Override + public Iterator iterator() { + return stats.iterator(); + } + + public static ThreadPoolStats readThreadPoolStats(StreamInput in) throws IOException { + ThreadPoolStats stats = new ThreadPoolStats(); + stats.readFrom(in); + return stats; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + int size = in.readVInt(); + stats = new ArrayList(size); + for (int i = 0; i < size; i++) { + Stats stats1 = new Stats(); + stats1.readFrom(in); + stats.add(stats1); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(stats.size()); + for (Stats stat : stats) { + stat.writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject("thread_pool"); + for (Stats stat : stats) { + stat.toXContent(builder, params); + } + builder.endObject(); + return builder; + } +} From 771dbdb4bc007663bfdc234a36327736c957ae32 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 11 Jan 2012 15:01:40 +0200 Subject: [PATCH 155/270] doc nested docs and get / uid --- src/main/java/org/elasticsearch/common/lucene/uid/UidField.java | 2 ++ .../test/integration/nested/SimpleNestedTests.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/main/java/org/elasticsearch/common/lucene/uid/UidField.java b/src/main/java/org/elasticsearch/common/lucene/uid/UidField.java index 0e027fbd282..885328f068b 100644 --- a/src/main/java/org/elasticsearch/common/lucene/uid/UidField.java +++ b/src/main/java/org/elasticsearch/common/lucene/uid/UidField.java @@ -48,6 +48,8 @@ public class UidField extends AbstractField { } } + // this works fine for nested docs since they don't have the payload which has the version + // so we iterate till we find the one with the payload public static DocIdAndVersion loadDocIdAndVersion(IndexReader reader, Term term) { int docId = Lucene.NO_DOC; TermPositions uid = null; diff --git a/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java b/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java index 499700e4e75..a474dcbd3e2 100644 --- a/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java +++ b/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java @@ -41,6 +41,7 @@ import static org.elasticsearch.index.query.FilterBuilders.nestedFilter; import static org.elasticsearch.index.query.QueryBuilders.*; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; @Test public class SimpleNestedTests extends AbstractNodesTests { @@ -99,6 +100,7 @@ public class SimpleNestedTests extends AbstractNodesTests { client.admin().indices().prepareFlush().setRefresh(true).execute().actionGet(); GetResponse getResponse = client.prepareGet("test", "type1", "1").execute().actionGet(); assertThat(getResponse.exists(), equalTo(true)); + assertThat(getResponse.source(), notNullValue()); // check the numDocs IndicesStatusResponse statusResponse = client.admin().indices().prepareStatus().execute().actionGet(); From 002c8a65992190f2945e30e3b8a8cf8aabec9dab Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 11 Jan 2012 15:39:09 +0200 Subject: [PATCH 156/270] upgrade to 11.0.1 guava --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d759a86378b..817f98c9b46 100644 --- a/pom.xml +++ b/pom.xml @@ -84,7 +84,7 @@ com.google.guava guava - 11.0 + 11.0.1 compile From a380e0e16910d54fc6b098373be55e75ff3b3062 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 12 Jan 2012 13:39:08 +0200 Subject: [PATCH 157/270] optimize match all query against a type to be wrapped with a constant score query and not filtered --- .../java/org/elasticsearch/search/query/QueryPhase.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 5311551ca64..31021f33f03 100644 --- a/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.*; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.lucene.search.function.BoostScoreFunction; import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; import org.elasticsearch.index.query.ParsedQuery; @@ -81,7 +82,13 @@ public class QueryPhase implements SearchPhase { } Filter searchFilter = context.mapperService().searchFilter(context.types()); if (searchFilter != null) { - context.parsedQuery(new ParsedQuery(new FilteredQuery(context.query(), context.filterCache().cache(searchFilter)), context.parsedQuery())); + if (Queries.isMatchAllQuery(context.query())) { + Query q = new DeletionAwareConstantScoreQuery(context.filterCache().cache(searchFilter)); + q.setBoost(context.query().getBoost()); + context.parsedQuery(new ParsedQuery(q, context.parsedQuery())); + } else { + context.parsedQuery(new ParsedQuery(new FilteredQuery(context.query(), context.filterCache().cache(searchFilter)), context.parsedQuery())); + } } facetPhase.preProcess(context); } From 04a138db5ddf3f28ffd0d7f81054b05c3c6aa02d Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 12 Jan 2012 14:19:21 +0200 Subject: [PATCH 158/270] Allow to provide timeout parameter in request body (as well as URI parameter), closes #1604. --- .../action/search/SearchRequest.java | 34 --------- .../search/type/TransportSearchHelper.java | 1 - .../action/search/SearchRequestBuilder.java | 4 +- .../rest/action/search/RestSearchAction.java | 7 +- .../search/builder/SearchSourceBuilder.java | 23 ++++++ .../search/internal/ContextIndexSearcher.java | 7 +- .../internal/InternalSearchRequest.java | 22 ------ .../search/internal/SearchContext.java | 12 +++- .../search/query/QueryPhase.java | 2 + .../search/query/TimeoutParseElement.java | 41 +++++++++++ .../search/timeout/SearchTimeoutTests.java | 71 +++++++++++++++++++ 11 files changed, 157 insertions(+), 67 deletions(-) create mode 100644 src/main/java/org/elasticsearch/search/query/TimeoutParseElement.java create mode 100644 src/test/java/org/elasticsearch/test/integration/search/timeout/SearchTimeoutTests.java diff --git a/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 5dc611a78e6..2727b8df6bc 100644 --- a/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -44,7 +44,6 @@ import java.util.Arrays; import java.util.Map; import static org.elasticsearch.action.Actions.addValidationError; -import static org.elasticsearch.common.unit.TimeValue.readTimeValue; import static org.elasticsearch.search.Scroll.readScroll; /** @@ -89,8 +88,6 @@ public class SearchRequest implements ActionRequest { private String[] types = Strings.EMPTY_ARRAY; - private TimeValue timeout; - private boolean listenerThreaded = false; private SearchOperationThreading operationThreading = SearchOperationThreading.THREAD_PER_SHARD; @@ -501,28 +498,6 @@ public class SearchRequest implements ActionRequest { return scroll(new Scroll(TimeValue.parseTimeValue(keepAlive, null))); } - /** - * An optional timeout to control how long search is allowed to take. - */ - public TimeValue timeout() { - return timeout; - } - - /** - * An optional timeout to control how long search is allowed to take. - */ - public SearchRequest timeout(TimeValue timeout) { - this.timeout = timeout; - return this; - } - - /** - * An optional timeout to control how long search is allowed to take. - */ - public SearchRequest timeout(String timeout) { - return timeout(TimeValue.parseTimeValue(timeout, null)); - } - @Override public void readFrom(StreamInput in) throws IOException { operationThreading = SearchOperationThreading.fromId(in.readByte()); @@ -546,9 +521,6 @@ public class SearchRequest implements ActionRequest { if (in.readBoolean()) { scroll = readScroll(in); } - if (in.readBoolean()) { - timeout = readTimeValue(in); - } BytesHolder bytes = in.readBytesReference(); sourceUnsafe = false; @@ -606,12 +578,6 @@ public class SearchRequest implements ActionRequest { out.writeBoolean(true); scroll.writeTo(out); } - if (timeout == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - timeout.writeTo(out); - } out.writeBytesHolder(source, sourceOffset, sourceLength); out.writeBytesHolder(extraSource, extraSourceOffset, extraSourceLength); out.writeVInt(types.length); diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java index 3cd71fd8231..c7425f1942b 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java @@ -66,7 +66,6 @@ public abstract class TransportSearchHelper { internalRequest.source(request.source(), request.sourceOffset(), request.sourceLength()); internalRequest.extraSource(request.extraSource(), request.extraSourceOffset(), request.extraSourceLength()); internalRequest.scroll(request.scroll()); - internalRequest.timeout(request.timeout()); internalRequest.filteringAliases(filteringAliases); internalRequest.types(request.types()); internalRequest.nowInMillis(nowInMillis); diff --git a/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java b/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java index ca85f603bb6..6428625162c 100644 --- a/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java +++ b/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java @@ -115,7 +115,7 @@ public class SearchRequestBuilder extends BaseRequestBuilder fieldNames; private List scriptFields; private List partialFields; @@ -276,6 +279,22 @@ public class SearchSourceBuilder implements ToXContent { return this; } + /** + * An optional timeout to control how long search is allowed to take. + */ + public SearchSourceBuilder timeout(TimeValue timeout) { + this.timeoutInMillis = timeout.millis(); + return this; + } + + /** + * An optional timeout to control how long search is allowed to take. + */ + public SearchSourceBuilder timeout(String timeout) { + this.timeoutInMillis = TimeValue.parseTimeValue(timeout, null).millis(); + return this; + } + /** * Adds a sort against the given field name and the sort ordering. * @@ -564,6 +583,10 @@ public class SearchSourceBuilder implements ToXContent { builder.field("size", size); } + if (timeoutInMillis != -1) { + builder.field("timeout", timeoutInMillis); + } + if (queryBuilder != null) { builder.field("query"); queryBuilder.toXContent(builder, params); diff --git a/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 7f19148d2cb..91e6bc4a858 100644 --- a/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -158,9 +158,9 @@ public class ContextIndexSearcher extends ExtendedIndexSearcher { // since that is where the filter should only work collector = new FilteredCollector(collector, searchContext.parsedFilter()); } - if (searchContext.timeout() != null) { + if (searchContext.timeoutInMillis() != -1) { // TODO: change to use our own counter that uses the scheduler in ThreadPool - collector = new TimeLimitingCollector(collector, TimeLimitingCollector.getGlobalCounter(), searchContext.timeout().millis()); + collector = new TimeLimitingCollector(collector, TimeLimitingCollector.getGlobalCounter(), searchContext.timeoutInMillis()); } if (scopeCollectors != null) { List collectors = scopeCollectors.get(processingScope); @@ -185,8 +185,7 @@ public class ContextIndexSearcher extends ExtendedIndexSearcher { } // we only compute the doc id set once since within a context, we execute the same query always... - if (searchContext.timeout() != null) { - searchContext.queryResult().searchTimedOut(false); + if (searchContext.timeoutInMillis() != -1) { try { super.search(weight, combinedFilter, collector); } catch (TimeLimitingCollector.TimeExceededException e) { diff --git a/src/main/java/org/elasticsearch/search/internal/InternalSearchRequest.java b/src/main/java/org/elasticsearch/search/internal/InternalSearchRequest.java index 66aef768097..bb2b5ba22c9 100644 --- a/src/main/java/org/elasticsearch/search/internal/InternalSearchRequest.java +++ b/src/main/java/org/elasticsearch/search/internal/InternalSearchRequest.java @@ -26,12 +26,10 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.Scroll; import java.io.IOException; -import static org.elasticsearch.common.unit.TimeValue.readTimeValue; import static org.elasticsearch.search.Scroll.readScroll; /** @@ -63,8 +61,6 @@ public class InternalSearchRequest implements Streamable { private Scroll scroll; - private TimeValue timeout; - private String[] types = Strings.EMPTY_ARRAY; private String[] filteringAliases; @@ -169,15 +165,6 @@ public class InternalSearchRequest implements Streamable { return this; } - public TimeValue timeout() { - return timeout; - } - - public InternalSearchRequest timeout(TimeValue timeout) { - this.timeout = timeout; - return this; - } - public String[] filteringAliases() { return filteringAliases; } @@ -203,9 +190,6 @@ public class InternalSearchRequest implements Streamable { if (in.readBoolean()) { scroll = readScroll(in); } - if (in.readBoolean()) { - timeout = readTimeValue(in); - } BytesHolder bytes = in.readBytesReference(); source = bytes.bytes(); @@ -248,12 +232,6 @@ public class InternalSearchRequest implements Streamable { out.writeBoolean(true); scroll.writeTo(out); } - if (timeout == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - timeout.writeTo(out); - } out.writeBytesHolder(source, sourceOffset, sourceLength); out.writeBytesHolder(extraSource, extraSourceOffset, extraSourceLength); out.writeVInt(types.length); diff --git a/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 68e644542d0..ba563034294 100644 --- a/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -28,7 +28,6 @@ import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.lease.Releasable; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.cache.filter.FilterCache; @@ -112,6 +111,9 @@ public class SearchContext implements Releasable { private float queryBoost = 1.0f; + // timeout in millis + private long timeoutInMillis = -1; + private List groupStats; @@ -337,8 +339,12 @@ public class SearchContext implements Releasable { return indexService.cache().idCache(); } - public TimeValue timeout() { - return request.timeout(); + public long timeoutInMillis() { + return timeoutInMillis; + } + + public void timeoutInMillis(long timeoutInMillis) { + this.timeoutInMillis = timeoutInMillis; } public SearchContext minimumScore(float minimumScore) { diff --git a/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 31021f33f03..87bb304849e 100644 --- a/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -68,6 +68,7 @@ public class QueryPhase implements SearchPhase { .put("track_scores", new TrackScoresParseElement()) .put("min_score", new MinScoreParseElement()) .put("minScore", new MinScoreParseElement()) + .put("timeout", new TimeoutParseElement()) .putAll(facetPhase.parseElements()); return parseElements.build(); } @@ -94,6 +95,7 @@ public class QueryPhase implements SearchPhase { } public void execute(SearchContext searchContext) throws QueryPhaseExecutionException { + searchContext.queryResult().searchTimedOut(false); // set the filter on the searcher if (searchContext.scopePhases() != null) { // we have scoped queries, refresh the id cache diff --git a/src/main/java/org/elasticsearch/search/query/TimeoutParseElement.java b/src/main/java/org/elasticsearch/search/query/TimeoutParseElement.java new file mode 100644 index 00000000000..5c9acb567c2 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/query/TimeoutParseElement.java @@ -0,0 +1,41 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.query; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchParseElement; +import org.elasticsearch.search.internal.SearchContext; + +/** + */ +public class TimeoutParseElement implements SearchParseElement { + + @Override + public void parse(XContentParser parser, SearchContext context) throws Exception { + XContentParser.Token token = parser.currentToken(); + if (token == XContentParser.Token.VALUE_NUMBER) { + context.timeoutInMillis(parser.longValue()); + } else { + context.timeoutInMillis(TimeValue.parseTimeValue(parser.text(), null).millis()); + } + } +} + diff --git a/src/test/java/org/elasticsearch/test/integration/search/timeout/SearchTimeoutTests.java b/src/test/java/org/elasticsearch/test/integration/search/timeout/SearchTimeoutTests.java new file mode 100644 index 00000000000..66d58beb386 --- /dev/null +++ b/src/test/java/org/elasticsearch/test/integration/search/timeout/SearchTimeoutTests.java @@ -0,0 +1,71 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.integration.search.timeout; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.integration.AbstractNodesTests; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; +import static org.elasticsearch.index.query.FilterBuilders.scriptFilter; +import static org.elasticsearch.index.query.QueryBuilders.filteredQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +/** + */ +public class SearchTimeoutTests extends AbstractNodesTests { + + private Client client; + + @BeforeClass + public void createNodes() throws Exception { + Settings settings = settingsBuilder().put("index.number_of_shards", 2).put("index.number_of_replicas", 0).build(); + startNode("node1", settings); + client = client("node1"); + } + + @AfterClass + public void closeNodes() { + client.close(); + closeAllNodes(); + } + + @Test + public void simpleTimeoutTest() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + + for (int i = 0; i < 10; i++) { + client.prepareIndex("test", "type", Integer.toString(i)).setSource("field", "value").execute().actionGet(); + } + client.admin().indices().prepareRefresh().execute().actionGet(); + + SearchResponse searchResponse = client.prepareSearch("test") + .setTimeout("10ms") + .setQuery(filteredQuery(matchAllQuery(), scriptFilter("Thread.sleep(100); return true;"))) + .execute().actionGet(); + assertThat(searchResponse.timedOut(), equalTo(true)); + } +} From 97b5c100ce245168b01b0f126fc6c7cff54d8be7 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 12 Jan 2012 14:31:37 +0200 Subject: [PATCH 159/270] Add a simplified setting to disable shutdown API: action.disable_shutdown, closes #1606. --- .../cluster/node/shutdown/TransportNodesShutdownAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportNodesShutdownAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportNodesShutdownAction.java index bc8debcf44d..148766484b2 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportNodesShutdownAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportNodesShutdownAction.java @@ -62,7 +62,7 @@ public class TransportNodesShutdownAction extends TransportMasterNodeOperationAc super(settings, transportService, clusterService, threadPool); this.node = node; this.clusterName = clusterName; - this.disabled = componentSettings.getAsBoolean("disabled", false); + this.disabled = settings.getAsBoolean("action.disable_shutdown", componentSettings.getAsBoolean("disabled", false)); this.delay = componentSettings.getAsTime("delay", TimeValue.timeValueMillis(200)); this.transportService.registerHandler(NodeShutdownRequestHandler.ACTION, new NodeShutdownRequestHandler()); From 8f0ecbcc0b0148c011b66332d42791eefc9e580b Mon Sep 17 00:00:00 2001 From: Olivier Favre Date: Tue, 10 Jan 2012 17:21:54 +0100 Subject: [PATCH 160/270] Improve latitude and longitude normalization --- .../index/search/geo/GeoUtils.java | 36 ++-- .../unit/index/search/geo/GeoUtilsTests.java | 177 ++++++++++++++++++ 2 files changed, 189 insertions(+), 24 deletions(-) create mode 100644 src/test/java/org/elasticsearch/test/unit/index/search/geo/GeoUtilsTests.java diff --git a/src/main/java/org/elasticsearch/index/search/geo/GeoUtils.java b/src/main/java/org/elasticsearch/index/search/geo/GeoUtils.java index 69d560dcc7e..493943f65b2 100644 --- a/src/main/java/org/elasticsearch/index/search/geo/GeoUtils.java +++ b/src/main/java/org/elasticsearch/index/search/geo/GeoUtils.java @@ -24,32 +24,20 @@ package org.elasticsearch.index.search.geo; public class GeoUtils { public static double normalizeLon(double lon) { - double delta = 0; - if (lon < 0) { - delta = 360; - } else if (lon >= 0) { - delta = -360; - } - - double newLng = lon; - while (newLng < -180 || newLng > 180) { - newLng += delta; - } - return newLng; + return centeredModulus(lon, 360); } public static double normalizeLat(double lat) { - double delta = 0; - if (lat < 0) { - delta = 180; - } else if (lat >= 0) { - delta = -180; - } - - double newLat = lat; - while (newLat < -90 || newLat > 90) { - newLat += delta; - } - return newLat; + return centeredModulus(lat, 180); } + + private static double centeredModulus(double dividend, double divisor) { + double rtn = dividend % divisor; + if (rtn <= 0) + rtn += divisor; + if (rtn > divisor/2) + rtn -= divisor; + return rtn; + } + } diff --git a/src/test/java/org/elasticsearch/test/unit/index/search/geo/GeoUtilsTests.java b/src/test/java/org/elasticsearch/test/unit/index/search/geo/GeoUtilsTests.java new file mode 100644 index 00000000000..430142712d8 --- /dev/null +++ b/src/test/java/org/elasticsearch/test/unit/index/search/geo/GeoUtilsTests.java @@ -0,0 +1,177 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.unit.index.search.geo; + +import org.elasticsearch.index.search.geo.GeoUtils; +import org.testng.annotations.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +/** + * + */ +public class GeoUtilsTests { + + /** + * Test special values like inf, NaN and -0.0. + */ + @Test + public void testSpecials() { + assertThat(GeoUtils.normalizeLon(Double.POSITIVE_INFINITY), equalTo(Double.NaN)); + assertThat(GeoUtils.normalizeLat(Double.POSITIVE_INFINITY), equalTo(Double.NaN)); + assertThat(GeoUtils.normalizeLon(Double.NEGATIVE_INFINITY), equalTo(Double.NaN)); + assertThat(GeoUtils.normalizeLat(Double.NEGATIVE_INFINITY), equalTo(Double.NaN)); + assertThat(GeoUtils.normalizeLon(Double.NaN), equalTo(Double.NaN)); + assertThat(GeoUtils.normalizeLat(Double.NaN), equalTo(Double.NaN)); + assertThat(0.0, not(equalTo(-0.0))); + assertThat(GeoUtils.normalizeLon(-0.0), equalTo(0.0)); + assertThat(GeoUtils.normalizeLat(-0.0), equalTo(0.0)); + assertThat(GeoUtils.normalizeLon( 0.0), equalTo(0.0)); + assertThat(GeoUtils.normalizeLat( 0.0), equalTo(0.0)); + } + + /** + * Test bounding values. + */ + @Test + public void testBounds() { + assertThat(GeoUtils.normalizeLon(-360.0), equalTo(0.0)); + assertThat(GeoUtils.normalizeLat(-180.0), equalTo(0.0)); + assertThat(GeoUtils.normalizeLon( 360.0), equalTo(0.0)); + assertThat(GeoUtils.normalizeLat( 180.0), equalTo(0.0)); + // and halves + assertThat(GeoUtils.normalizeLon(-180.0), equalTo(180.0)); + assertThat(GeoUtils.normalizeLat(- 90.0), equalTo( 90.0)); + assertThat(GeoUtils.normalizeLon( 180.0), equalTo(180.0)); + assertThat(GeoUtils.normalizeLat( 90.0), equalTo( 90.0)); + } + + /** + * Test normal values. + */ + @Test + public void testNormal() { + // Near bounds + assertThat(GeoUtils.normalizeLon(-360.5), equalTo(-0.5)); + assertThat(GeoUtils.normalizeLat(-180.5), equalTo(-0.5)); + assertThat(GeoUtils.normalizeLon( 360.5), equalTo( 0.5)); + assertThat(GeoUtils.normalizeLat( 180.5), equalTo( 0.5)); + // and near halves + assertThat(GeoUtils.normalizeLon(-180.5), equalTo( 179.5)); + assertThat(GeoUtils.normalizeLat(- 90.5), equalTo( 89.5)); + assertThat(GeoUtils.normalizeLon( 180.5), equalTo(-179.5)); + assertThat(GeoUtils.normalizeLat( 90.5), equalTo(- 89.5)); + // Every 10-units, multiple full turns + for (int shift = -20 ; shift <= 20 ; ++shift) { + assertThat(GeoUtils.normalizeLon(shift*360.0+ 0.0), equalTo( 0.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 10.0), equalTo( 10.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 20.0), equalTo( 20.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 30.0), equalTo( 30.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 40.0), equalTo( 40.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 50.0), equalTo( 50.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 60.0), equalTo( 60.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 70.0), equalTo( 70.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 80.0), equalTo( 80.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 90.0), equalTo( 90.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 100.0), equalTo( 100.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 110.0), equalTo( 110.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 120.0), equalTo( 120.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 130.0), equalTo( 130.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 140.0), equalTo( 140.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 150.0), equalTo( 150.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 160.0), equalTo( 160.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 170.0), equalTo( 170.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 180.0), equalTo( 180.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 190.0), equalTo(-170.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 200.0), equalTo(-160.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 210.0), equalTo(-150.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 220.0), equalTo(-140.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 230.0), equalTo(-130.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 240.0), equalTo(-120.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 250.0), equalTo(-110.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 260.0), equalTo(-100.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 270.0), equalTo(- 90.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 280.0), equalTo(- 80.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 290.0), equalTo(- 70.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 300.0), equalTo(- 60.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 310.0), equalTo(- 50.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 320.0), equalTo(- 40.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 330.0), equalTo(- 30.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 340.0), equalTo(- 20.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 350.0), equalTo(- 10.0)); + assertThat(GeoUtils.normalizeLon(shift*360.0+ 360.0), equalTo( 0.0)); + } + for (int shift = -20 ; shift <= 20 ; ++shift) { + assertThat(GeoUtils.normalizeLat(shift*180.0+ 0.0), equalTo( 0.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 10.0), equalTo( 10.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 20.0), equalTo( 20.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 30.0), equalTo( 30.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 40.0), equalTo( 40.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 50.0), equalTo( 50.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 60.0), equalTo( 60.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 70.0), equalTo( 70.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 80.0), equalTo( 80.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 90.0), equalTo( 90.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 100.0), equalTo(-80.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 110.0), equalTo(-70.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 120.0), equalTo(-60.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 130.0), equalTo(-50.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 140.0), equalTo(-40.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 150.0), equalTo(-30.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 160.0), equalTo(-20.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 170.0), equalTo(-10.0)); + assertThat(GeoUtils.normalizeLat(shift*180.0+ 180.0), equalTo( 0.0)); + } + } + + /** + * Test huge values. + */ + @Test + public void testHuge() { + assertThat(GeoUtils.normalizeLon(-36000000000181.0), equalTo(GeoUtils.normalizeLon(-181.0))); + assertThat(GeoUtils.normalizeLon(-36000000000180.0), equalTo(GeoUtils.normalizeLon(-180.0))); + assertThat(GeoUtils.normalizeLon(-36000000000179.0), equalTo(GeoUtils.normalizeLon(-179.0))); + assertThat(GeoUtils.normalizeLon(-36000000000178.0), equalTo(GeoUtils.normalizeLon(-178.0))); + assertThat(GeoUtils.normalizeLon(-36000000000001.0), equalTo(GeoUtils.normalizeLon(-001.0))); + assertThat(GeoUtils.normalizeLon(+36000000000000.0), equalTo(GeoUtils.normalizeLon(+000.0))); + assertThat(GeoUtils.normalizeLon(+36000000000001.0), equalTo(GeoUtils.normalizeLon(+001.0))); + assertThat(GeoUtils.normalizeLon(+36000000000002.0), equalTo(GeoUtils.normalizeLon(+002.0))); + assertThat(GeoUtils.normalizeLon(+36000000000178.0), equalTo(GeoUtils.normalizeLon(+178.0))); + assertThat(GeoUtils.normalizeLon(+36000000000179.0), equalTo(GeoUtils.normalizeLon(+179.0))); + assertThat(GeoUtils.normalizeLon(+36000000000180.0), equalTo(GeoUtils.normalizeLon(+180.0))); + assertThat(GeoUtils.normalizeLon(+36000000000181.0), equalTo(GeoUtils.normalizeLon(+181.0))); + assertThat(GeoUtils.normalizeLat(-18000000000091.0), equalTo(GeoUtils.normalizeLat(-091.0))); + assertThat(GeoUtils.normalizeLat(-18000000000090.0), equalTo(GeoUtils.normalizeLat(-090.0))); + assertThat(GeoUtils.normalizeLat(-18000000000089.0), equalTo(GeoUtils.normalizeLat(-089.0))); + assertThat(GeoUtils.normalizeLat(-18000000000088.0), equalTo(GeoUtils.normalizeLat(-088.0))); + assertThat(GeoUtils.normalizeLat(-18000000000001.0), equalTo(GeoUtils.normalizeLat(-001.0))); + assertThat(GeoUtils.normalizeLat(+18000000000000.0), equalTo(GeoUtils.normalizeLat(+000.0))); + assertThat(GeoUtils.normalizeLat(+18000000000001.0), equalTo(GeoUtils.normalizeLat(+001.0))); + assertThat(GeoUtils.normalizeLat(+18000000000002.0), equalTo(GeoUtils.normalizeLat(+002.0))); + assertThat(GeoUtils.normalizeLat(+18000000000088.0), equalTo(GeoUtils.normalizeLat(+088.0))); + assertThat(GeoUtils.normalizeLat(+18000000000089.0), equalTo(GeoUtils.normalizeLat(+089.0))); + assertThat(GeoUtils.normalizeLat(+18000000000090.0), equalTo(GeoUtils.normalizeLat(+090.0))); + assertThat(GeoUtils.normalizeLat(+18000000000091.0), equalTo(GeoUtils.normalizeLat(+091.0))); + } + +} From d2d65f2f65eefbcf36125681b9d66620f2cc0afb Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 12 Jan 2012 16:59:43 +0200 Subject: [PATCH 161/270] add test marker on the class as well --- .../unit/index/search/geo/GeoUtilsTests.java | 143 +++++++++--------- 1 file changed, 72 insertions(+), 71 deletions(-) diff --git a/src/test/java/org/elasticsearch/test/unit/index/search/geo/GeoUtilsTests.java b/src/test/java/org/elasticsearch/test/unit/index/search/geo/GeoUtilsTests.java index 430142712d8..50c37e7a29d 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/search/geo/GeoUtilsTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/search/geo/GeoUtilsTests.java @@ -29,6 +29,7 @@ import static org.hamcrest.Matchers.not; /** * */ +@Test public class GeoUtilsTests { /** @@ -45,8 +46,8 @@ public class GeoUtilsTests { assertThat(0.0, not(equalTo(-0.0))); assertThat(GeoUtils.normalizeLon(-0.0), equalTo(0.0)); assertThat(GeoUtils.normalizeLat(-0.0), equalTo(0.0)); - assertThat(GeoUtils.normalizeLon( 0.0), equalTo(0.0)); - assertThat(GeoUtils.normalizeLat( 0.0), equalTo(0.0)); + assertThat(GeoUtils.normalizeLon(0.0), equalTo(0.0)); + assertThat(GeoUtils.normalizeLat(0.0), equalTo(0.0)); } /** @@ -56,13 +57,13 @@ public class GeoUtilsTests { public void testBounds() { assertThat(GeoUtils.normalizeLon(-360.0), equalTo(0.0)); assertThat(GeoUtils.normalizeLat(-180.0), equalTo(0.0)); - assertThat(GeoUtils.normalizeLon( 360.0), equalTo(0.0)); - assertThat(GeoUtils.normalizeLat( 180.0), equalTo(0.0)); + assertThat(GeoUtils.normalizeLon(360.0), equalTo(0.0)); + assertThat(GeoUtils.normalizeLat(180.0), equalTo(0.0)); // and halves assertThat(GeoUtils.normalizeLon(-180.0), equalTo(180.0)); - assertThat(GeoUtils.normalizeLat(- 90.0), equalTo( 90.0)); - assertThat(GeoUtils.normalizeLon( 180.0), equalTo(180.0)); - assertThat(GeoUtils.normalizeLat( 90.0), equalTo( 90.0)); + assertThat(GeoUtils.normalizeLat(-90.0), equalTo(90.0)); + assertThat(GeoUtils.normalizeLon(180.0), equalTo(180.0)); + assertThat(GeoUtils.normalizeLat(90.0), equalTo(90.0)); } /** @@ -73,73 +74,73 @@ public class GeoUtilsTests { // Near bounds assertThat(GeoUtils.normalizeLon(-360.5), equalTo(-0.5)); assertThat(GeoUtils.normalizeLat(-180.5), equalTo(-0.5)); - assertThat(GeoUtils.normalizeLon( 360.5), equalTo( 0.5)); - assertThat(GeoUtils.normalizeLat( 180.5), equalTo( 0.5)); + assertThat(GeoUtils.normalizeLon(360.5), equalTo(0.5)); + assertThat(GeoUtils.normalizeLat(180.5), equalTo(0.5)); // and near halves - assertThat(GeoUtils.normalizeLon(-180.5), equalTo( 179.5)); - assertThat(GeoUtils.normalizeLat(- 90.5), equalTo( 89.5)); - assertThat(GeoUtils.normalizeLon( 180.5), equalTo(-179.5)); - assertThat(GeoUtils.normalizeLat( 90.5), equalTo(- 89.5)); + assertThat(GeoUtils.normalizeLon(-180.5), equalTo(179.5)); + assertThat(GeoUtils.normalizeLat(-90.5), equalTo(89.5)); + assertThat(GeoUtils.normalizeLon(180.5), equalTo(-179.5)); + assertThat(GeoUtils.normalizeLat(90.5), equalTo(-89.5)); // Every 10-units, multiple full turns - for (int shift = -20 ; shift <= 20 ; ++shift) { - assertThat(GeoUtils.normalizeLon(shift*360.0+ 0.0), equalTo( 0.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 10.0), equalTo( 10.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 20.0), equalTo( 20.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 30.0), equalTo( 30.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 40.0), equalTo( 40.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 50.0), equalTo( 50.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 60.0), equalTo( 60.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 70.0), equalTo( 70.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 80.0), equalTo( 80.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 90.0), equalTo( 90.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 100.0), equalTo( 100.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 110.0), equalTo( 110.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 120.0), equalTo( 120.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 130.0), equalTo( 130.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 140.0), equalTo( 140.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 150.0), equalTo( 150.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 160.0), equalTo( 160.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 170.0), equalTo( 170.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 180.0), equalTo( 180.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 190.0), equalTo(-170.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 200.0), equalTo(-160.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 210.0), equalTo(-150.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 220.0), equalTo(-140.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 230.0), equalTo(-130.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 240.0), equalTo(-120.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 250.0), equalTo(-110.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 260.0), equalTo(-100.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 270.0), equalTo(- 90.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 280.0), equalTo(- 80.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 290.0), equalTo(- 70.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 300.0), equalTo(- 60.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 310.0), equalTo(- 50.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 320.0), equalTo(- 40.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 330.0), equalTo(- 30.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 340.0), equalTo(- 20.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 350.0), equalTo(- 10.0)); - assertThat(GeoUtils.normalizeLon(shift*360.0+ 360.0), equalTo( 0.0)); + for (int shift = -20; shift <= 20; ++shift) { + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 0.0), equalTo(0.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 10.0), equalTo(10.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 20.0), equalTo(20.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 30.0), equalTo(30.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 40.0), equalTo(40.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 50.0), equalTo(50.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 60.0), equalTo(60.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 70.0), equalTo(70.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 80.0), equalTo(80.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 90.0), equalTo(90.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 100.0), equalTo(100.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 110.0), equalTo(110.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 120.0), equalTo(120.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 130.0), equalTo(130.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 140.0), equalTo(140.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 150.0), equalTo(150.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 160.0), equalTo(160.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 170.0), equalTo(170.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 180.0), equalTo(180.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 190.0), equalTo(-170.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 200.0), equalTo(-160.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 210.0), equalTo(-150.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 220.0), equalTo(-140.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 230.0), equalTo(-130.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 240.0), equalTo(-120.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 250.0), equalTo(-110.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 260.0), equalTo(-100.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 270.0), equalTo(-90.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 280.0), equalTo(-80.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 290.0), equalTo(-70.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 300.0), equalTo(-60.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 310.0), equalTo(-50.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 320.0), equalTo(-40.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 330.0), equalTo(-30.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 340.0), equalTo(-20.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 350.0), equalTo(-10.0)); + assertThat(GeoUtils.normalizeLon(shift * 360.0 + 360.0), equalTo(0.0)); } - for (int shift = -20 ; shift <= 20 ; ++shift) { - assertThat(GeoUtils.normalizeLat(shift*180.0+ 0.0), equalTo( 0.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 10.0), equalTo( 10.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 20.0), equalTo( 20.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 30.0), equalTo( 30.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 40.0), equalTo( 40.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 50.0), equalTo( 50.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 60.0), equalTo( 60.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 70.0), equalTo( 70.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 80.0), equalTo( 80.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 90.0), equalTo( 90.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 100.0), equalTo(-80.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 110.0), equalTo(-70.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 120.0), equalTo(-60.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 130.0), equalTo(-50.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 140.0), equalTo(-40.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 150.0), equalTo(-30.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 160.0), equalTo(-20.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 170.0), equalTo(-10.0)); - assertThat(GeoUtils.normalizeLat(shift*180.0+ 180.0), equalTo( 0.0)); + for (int shift = -20; shift <= 20; ++shift) { + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 0.0), equalTo(0.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 10.0), equalTo(10.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 20.0), equalTo(20.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 30.0), equalTo(30.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 40.0), equalTo(40.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 50.0), equalTo(50.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 60.0), equalTo(60.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 70.0), equalTo(70.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 80.0), equalTo(80.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 90.0), equalTo(90.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 100.0), equalTo(-80.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 110.0), equalTo(-70.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 120.0), equalTo(-60.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 130.0), equalTo(-50.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 140.0), equalTo(-40.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 150.0), equalTo(-30.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 160.0), equalTo(-20.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 170.0), equalTo(-10.0)); + assertThat(GeoUtils.normalizeLat(shift * 180.0 + 180.0), equalTo(0.0)); } } From 5eedfb1d62c5c251de523cd2ead08ae1dc373301 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 13 Jan 2012 10:40:29 +0200 Subject: [PATCH 162/270] Multicast Discovery: if it fails, still start in a single cluster mode, closes #1608. --- .../zen/ping/multicast/MulticastZenPing.java | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java b/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java index 2693049d7b3..ef5affae086 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java @@ -21,6 +21,7 @@ package org.elasticsearch.discovery.zen.ping.multicast; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ElasticSearchIllegalStateException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -35,7 +36,6 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.discovery.zen.DiscoveryNodesProvider; import org.elasticsearch.discovery.zen.ping.ZenPing; -import org.elasticsearch.discovery.zen.ping.ZenPingException; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.*; @@ -239,14 +239,14 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem } final int id = pingIdGenerator.incrementAndGet(); receivedResponses.put(id, new ConcurrentHashMap()); - sendPingRequest(id, true); + sendPingRequest(id); // try and send another ping request halfway through (just in case someone woke up during it...) // this can be a good trade-off to nailing the initial lookup or un-delivered messages threadPool.schedule(TimeValue.timeValueMillis(timeout.millis() / 2), ThreadPool.Names.CACHED, new Runnable() { @Override public void run() { try { - sendPingRequest(id, false); + sendPingRequest(id); } catch (Exception e) { logger.warn("[{}] failed to send second ping request", e, id); } @@ -261,7 +261,7 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem }); } - private void sendPingRequest(int id, boolean remove) { + private void sendPingRequest(int id) { if (multicastSocket == null) { return; } @@ -275,27 +275,21 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem clusterName.writeTo(out); nodesProvider.nodes().localNode().writeTo(out); datagramPacketSend.setData(cachedEntry.bytes().copiedByteArray()); - } catch (IOException e) { - if (remove) { - receivedResponses.remove(id); - } - throw new ZenPingException("Failed to serialize ping request", e); - } finally { - CachedStreamOutput.pushEntry(cachedEntry); - } - try { multicastSocket.send(datagramPacketSend); if (logger.isTraceEnabled()) { logger.trace("[{}] sending ping request", id); } - } catch (IOException e) { - if (remove) { - receivedResponses.remove(id); - } + } catch (Exception e) { if (lifecycle.stoppedOrClosed()) { return; } - throw new ZenPingException("Failed to send ping request over multicast on " + multicastSocket, e); + if (logger.isDebugEnabled()) { + logger.debug("failed to send multicast ping request", e); + } else { + logger.warn("failed to send multicast ping request: {}", ExceptionsHelper.detailedMessage(e)); + } + } finally { + CachedStreamOutput.pushEntry(cachedEntry); } } } From 7a8e420363da86d178d34509e3344f437a5564e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Vl=C4=8Dek?= Date: Fri, 13 Jan 2012 14:25:13 +0100 Subject: [PATCH 163/270] fix typos in javadoc --- .../indices/template/put/PutIndexTemplateRequest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index 308ab2d0225..b3934931510 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -45,7 +45,7 @@ import static org.elasticsearch.common.settings.ImmutableSettings.writeSettingsT import static org.elasticsearch.common.unit.TimeValue.readTimeValue; /** - * + * A request to create an index template. */ public class PutIndexTemplateRequest extends MasterNodeOperationRequest { @@ -134,7 +134,7 @@ public class PutIndexTemplateRequest extends MasterNodeOperationRequest { } /** - * The settings to created the index template with. + * The settings to create the index template with. */ public PutIndexTemplateRequest settings(Settings settings) { this.settings = settings; @@ -142,7 +142,7 @@ public class PutIndexTemplateRequest extends MasterNodeOperationRequest { } /** - * The settings to created the index template with. + * The settings to create the index template with. */ public PutIndexTemplateRequest settings(Settings.Builder settings) { this.settings = settings.build(); @@ -150,7 +150,7 @@ public class PutIndexTemplateRequest extends MasterNodeOperationRequest { } /** - * The settings to crete the index template with (either json/yaml/properties format) + * The settings to crete the index template with (either json/yaml/properties format). */ public PutIndexTemplateRequest settings(String source) { this.settings = ImmutableSettings.settingsBuilder().loadFromSource(source).build(); @@ -158,7 +158,7 @@ public class PutIndexTemplateRequest extends MasterNodeOperationRequest { } /** - * The settings to crete the index template with (either json/yaml/properties format) + * The settings to crete the index template with (either json/yaml/properties format). */ public PutIndexTemplateRequest settings(Map source) { try { From 2c5824d5bf0259c5da34922d892e54b938e2b095 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 14 Jan 2012 00:25:24 +0200 Subject: [PATCH 164/270] first step into trying to allow plugins to define custom actions that are integrated into the Client interface by allowing to register custom actions with TransportActionModule and ClientTransportActionModule --- .../action/TransportActionModule.java | 172 +++++++++++------- .../action/TransportActions.java | 12 +- .../TransportDeleteIndexTemplateAction.java | 4 +- .../put/TransportPutIndexTemplateAction.java | 4 +- .../query/TransportValidateQueryAction.java | 2 +- .../elasticsearch/client/node/NodeClient.java | 34 ++-- .../client/node/NodeClusterAdminClient.java | 31 ++-- .../client/node/NodeIndicesAdminClient.java | 55 +++--- .../action/ClientTransportActionModule.java | 128 ++++++++----- ...entTransportDeleteIndexTemplateAction.java | 2 +- ...ClientTransportPutIndexTemplateAction.java | 2 +- .../ClientTransportValidateQueryAction.java | 2 +- .../support/InternalTransportClient.java | 33 ++-- .../InternalTransportClusterAdminClient.java | 30 +-- .../InternalTransportIndicesAdminClient.java | 55 +++--- 15 files changed, 322 insertions(+), 244 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/TransportActionModule.java b/src/main/java/org/elasticsearch/action/TransportActionModule.java index 334f1d0911a..5a6765df6bc 100644 --- a/src/main/java/org/elasticsearch/action/TransportActionModule.java +++ b/src/main/java/org/elasticsearch/action/TransportActionModule.java @@ -19,6 +19,7 @@ package org.elasticsearch.action; +import com.google.common.collect.Maps; import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction; import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; import org.elasticsearch.action.admin.cluster.node.restart.TransportNodesRestartAction; @@ -71,90 +72,125 @@ import org.elasticsearch.action.percolate.TransportPercolateAction; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.search.TransportSearchScrollAction; import org.elasticsearch.action.search.type.*; +import org.elasticsearch.action.support.BaseAction; import org.elasticsearch.action.update.TransportUpdateAction; import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.inject.multibindings.MapBinder; + +import java.util.Map; /** * */ public class TransportActionModule extends AbstractModule { + private final Map actions = Maps.newHashMap(); + + static class ActionEntry { + public final String name; + public final Class action; + public final Class[] supportActions; + + ActionEntry(String name, Class action, Class... supportActions) { + this.name = name; + this.action = action; + this.supportActions = supportActions; + } + + + } + + public TransportActionModule() { + } + + /** + * Registers a custom action under the provided action name, the actual action implementation, and + * any supported actions (bind as singletons). + * + * @param actionName The action name + * @param action The action itself + * @param supportActions Support actions. + */ + public void registerAction(String actionName, Class action, Class... supportActions) { + actions.put(actionName, new ActionEntry(actionName, action, supportActions)); + } + @Override protected void configure() { - bind(TransportNodesInfoAction.class).asEagerSingleton(); - bind(TransportNodesStatsAction.class).asEagerSingleton(); - bind(TransportNodesShutdownAction.class).asEagerSingleton(); - bind(TransportNodesRestartAction.class).asEagerSingleton(); - bind(TransportClusterStateAction.class).asEagerSingleton(); - bind(TransportClusterHealthAction.class).asEagerSingleton(); - bind(TransportClusterUpdateSettingsAction.class).asEagerSingleton(); - bind(TransportClusterRerouteAction.class).asEagerSingleton(); + registerAction(TransportActions.Admin.Cluster.Node.INFO, TransportNodesInfoAction.class); + registerAction(TransportActions.Admin.Cluster.Node.STATS, TransportNodesStatsAction.class); + registerAction(TransportActions.Admin.Cluster.Node.SHUTDOWN, TransportNodesShutdownAction.class); + registerAction(TransportActions.Admin.Cluster.Node.RESTART, TransportNodesRestartAction.class); - bind(TransportSinglePingAction.class).asEagerSingleton(); - bind(TransportBroadcastPingAction.class).asEagerSingleton(); - bind(TransportShardReplicationPingAction.class).asEagerSingleton(); - bind(TransportIndexReplicationPingAction.class).asEagerSingleton(); - bind(TransportReplicationPingAction.class).asEagerSingleton(); + registerAction(TransportActions.Admin.Cluster.STATE, TransportClusterStateAction.class); + registerAction(TransportActions.Admin.Cluster.HEALTH, TransportClusterHealthAction.class); + registerAction(TransportActions.Admin.Cluster.UPDATE_SETTINGS, TransportClusterUpdateSettingsAction.class); + registerAction(TransportActions.Admin.Cluster.REROUTE, TransportClusterRerouteAction.class); - bind(TransportIndicesStatsAction.class).asEagerSingleton(); - bind(TransportIndicesStatusAction.class).asEagerSingleton(); - bind(TransportIndicesSegmentsAction.class).asEagerSingleton(); - bind(TransportCreateIndexAction.class).asEagerSingleton(); - bind(TransportDeleteIndexAction.class).asEagerSingleton(); - bind(TransportOpenIndexAction.class).asEagerSingleton(); - bind(TransportCloseIndexAction.class).asEagerSingleton(); - bind(TransportIndicesExistsAction.class).asEagerSingleton(); - bind(TransportPutMappingAction.class).asEagerSingleton(); - bind(TransportDeleteMappingAction.class).asEagerSingleton(); - bind(TransportIndicesAliasesAction.class).asEagerSingleton(); - bind(TransportUpdateSettingsAction.class).asEagerSingleton(); - bind(TransportAnalyzeAction.class).asEagerSingleton(); - bind(TransportPutIndexTemplateAction.class).asEagerSingleton(); - bind(TransportDeleteIndexTemplateAction.class).asEagerSingleton(); - bind(TransportValidateQueryAction.class).asEagerSingleton(); + registerAction(TransportActions.Admin.Cluster.Ping.SINGLE, TransportSinglePingAction.class); + registerAction(TransportActions.Admin.Cluster.Ping.BROADCAST, TransportBroadcastPingAction.class); + registerAction(TransportActions.Admin.Cluster.Ping.REPLICATION, TransportReplicationPingAction.class, + TransportIndexReplicationPingAction.class, TransportShardReplicationPingAction.class); - bind(TransportGatewaySnapshotAction.class).asEagerSingleton(); + registerAction(TransportActions.Admin.Indices.STATS, TransportIndicesStatsAction.class); + registerAction(TransportActions.Admin.Indices.STATUS, TransportIndicesStatusAction.class); + registerAction(TransportActions.Admin.Indices.SEGMENTS, TransportIndicesSegmentsAction.class); + registerAction(TransportActions.Admin.Indices.CREATE, TransportCreateIndexAction.class); + registerAction(TransportActions.Admin.Indices.DELETE, TransportDeleteIndexAction.class); + registerAction(TransportActions.Admin.Indices.OPEN, TransportOpenIndexAction.class); + registerAction(TransportActions.Admin.Indices.CLOSE, TransportCloseIndexAction.class); + registerAction(TransportActions.Admin.Indices.EXISTS, TransportIndicesExistsAction.class); + registerAction(TransportActions.Admin.Indices.Mapping.PUT, TransportPutMappingAction.class); + registerAction(TransportActions.Admin.Indices.Mapping.DELETE, TransportDeleteMappingAction.class); + registerAction(TransportActions.Admin.Indices.ALIASES, TransportIndicesAliasesAction.class); + registerAction(TransportActions.Admin.Indices.UPDATE_SETTINGS, TransportUpdateSettingsAction.class); + registerAction(TransportActions.Admin.Indices.ANALYZE, TransportAnalyzeAction.class); + registerAction(TransportActions.Admin.Indices.Template.PUT, TransportPutIndexTemplateAction.class); + registerAction(TransportActions.Admin.Indices.Template.DELETE, TransportDeleteIndexTemplateAction.class); + registerAction(TransportActions.Admin.Indices.Validate.QUERY, TransportValidateQueryAction.class); + registerAction(TransportActions.Admin.Indices.Gateway.SNAPSHOT, TransportGatewaySnapshotAction.class); + registerAction(TransportActions.Admin.Indices.REFRESH, TransportRefreshAction.class); + registerAction(TransportActions.Admin.Indices.FLUSH, TransportFlushAction.class); + registerAction(TransportActions.Admin.Indices.OPTIMIZE, TransportOptimizeAction.class); + registerAction(TransportActions.Admin.Indices.Cache.CLEAR, TransportClearIndicesCacheAction.class); - bind(TransportRefreshAction.class).asEagerSingleton(); - bind(TransportFlushAction.class).asEagerSingleton(); - bind(TransportOptimizeAction.class).asEagerSingleton(); - bind(TransportClearIndicesCacheAction.class).asEagerSingleton(); + registerAction(TransportActions.INDEX, TransportIndexAction.class); + registerAction(TransportActions.GET, TransportGetAction.class); + registerAction(TransportActions.DELETE, TransportDeleteAction.class, + TransportIndexDeleteAction.class, TransportShardDeleteAction.class); + registerAction(TransportActions.COUNT, TransportCountAction.class); + registerAction(TransportActions.UPDATE, TransportUpdateAction.class); + registerAction(TransportActions.MULTI_GET, TransportMultiGetAction.class, + TransportShardMultiGetAction.class); + registerAction(TransportActions.BULK, TransportBulkAction.class, + TransportShardBulkAction.class); + registerAction(TransportActions.DELETE_BY_QUERY, TransportDeleteByQueryAction.class, + TransportIndexDeleteByQueryAction.class, TransportShardDeleteByQueryAction.class); + registerAction(TransportActions.SEARCH, TransportSearchAction.class, + TransportSearchCache.class, + TransportSearchDfsQueryThenFetchAction.class, + TransportSearchQueryThenFetchAction.class, + TransportSearchDfsQueryAndFetchAction.class, + TransportSearchQueryAndFetchAction.class, + TransportSearchScanAction.class + ); + registerAction(TransportActions.SEARCH_SCROLL, TransportSearchScrollAction.class, + TransportSearchScrollScanAction.class, + TransportSearchScrollQueryThenFetchAction.class, + TransportSearchScrollQueryAndFetchAction.class + ); + registerAction(TransportActions.MORE_LIKE_THIS, TransportMoreLikeThisAction.class); + registerAction(TransportActions.PERCOLATE, TransportPercolateAction.class); - bind(TransportIndexAction.class).asEagerSingleton(); - bind(TransportGetAction.class).asEagerSingleton(); - bind(TransportDeleteAction.class).asEagerSingleton(); - bind(TransportIndexDeleteAction.class).asEagerSingleton(); - bind(TransportShardDeleteAction.class).asEagerSingleton(); - bind(TransportCountAction.class).asEagerSingleton(); - bind(TransportUpdateAction.class).asEagerSingleton(); + MapBinder actionsBinder + = MapBinder.newMapBinder(binder(), String.class, BaseAction.class); - bind(TransportMultiGetAction.class).asEagerSingleton(); - bind(TransportShardMultiGetAction.class).asEagerSingleton(); - - bind(TransportBulkAction.class).asEagerSingleton(); - bind(TransportShardBulkAction.class).asEagerSingleton(); - - bind(TransportShardDeleteByQueryAction.class).asEagerSingleton(); - bind(TransportIndexDeleteByQueryAction.class).asEagerSingleton(); - bind(TransportDeleteByQueryAction.class).asEagerSingleton(); - - - bind(TransportSearchCache.class).asEagerSingleton(); - bind(TransportSearchDfsQueryThenFetchAction.class).asEagerSingleton(); - bind(TransportSearchQueryThenFetchAction.class).asEagerSingleton(); - bind(TransportSearchDfsQueryAndFetchAction.class).asEagerSingleton(); - bind(TransportSearchQueryAndFetchAction.class).asEagerSingleton(); - bind(TransportSearchScanAction.class).asEagerSingleton(); - bind(TransportSearchAction.class).asEagerSingleton(); - - bind(TransportSearchScrollScanAction.class).asEagerSingleton(); - bind(TransportSearchScrollQueryThenFetchAction.class).asEagerSingleton(); - bind(TransportSearchScrollQueryAndFetchAction.class).asEagerSingleton(); - bind(TransportSearchScrollAction.class).asEagerSingleton(); - - bind(TransportMoreLikeThisAction.class).asEagerSingleton(); - - bind(TransportPercolateAction.class).asEagerSingleton(); + for (Map.Entry entry : actions.entrySet()) { + actionsBinder.addBinding(entry.getKey()).to(entry.getValue().action).asEagerSingleton(); + for (Class supportAction : entry.getValue().supportActions) { + bind(supportAction).asEagerSingleton(); + } + } } } diff --git a/src/main/java/org/elasticsearch/action/TransportActions.java b/src/main/java/org/elasticsearch/action/TransportActions.java index f98af2c008a..8f9d49d688c 100644 --- a/src/main/java/org/elasticsearch/action/TransportActions.java +++ b/src/main/java/org/elasticsearch/action/TransportActions.java @@ -54,9 +54,6 @@ public class TransportActions { public static final String ALIASES = "indices/aliases"; public static final String UPDATE_SETTINGS = "indices/updateSettings"; public static final String ANALYZE = "indices/analyze"; - public static final String PUT_INDEX_TEMPLATE = "indices/putIndexTemplate"; - public static final String DELETE_INDEX_TEMPLATE = "indices/deleteIndexTemplate"; - public static final String VALIDATE_QUERY = "indices/validateQuery"; public static class Gateway { public static final String SNAPSHOT = "indices/gateway/snapshot"; @@ -67,6 +64,15 @@ public class TransportActions { public static final String DELETE = "indices/mapping/delete"; } + public static class Template { + public static final String PUT = "indices/template/put"; + public static final String DELETE = "indices/template/delete"; + } + + public static class Validate { + public static final String QUERY = "indices/validate/query"; + } + public static class Cache { public static final String CLEAR = "indices/cache/clear"; } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java b/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java index 2ac0a23aa1c..2e9d8031450 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java @@ -37,8 +37,6 @@ import java.util.concurrent.atomic.AtomicReference; /** * Delete index action. - * - * */ public class TransportDeleteIndexTemplateAction extends TransportMasterNodeOperationAction { @@ -58,7 +56,7 @@ public class TransportDeleteIndexTemplateAction extends TransportMasterNodeOpera @Override protected String transportAction() { - return TransportActions.Admin.Indices.DELETE_INDEX_TEMPLATE; + return TransportActions.Admin.Indices.Template.DELETE; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java index 4b9d0887fb9..234618901f6 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java @@ -37,8 +37,6 @@ import java.util.concurrent.atomic.AtomicReference; /** * Put index template action. - * - * */ public class TransportPutIndexTemplateAction extends TransportMasterNodeOperationAction { @@ -58,7 +56,7 @@ public class TransportPutIndexTemplateAction extends TransportMasterNodeOperatio @Override protected String transportAction() { - return TransportActions.Admin.Indices.PUT_INDEX_TEMPLATE; + return TransportActions.Admin.Indices.Template.PUT; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java index a8fec5bce64..89e86ea901e 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java @@ -67,7 +67,7 @@ public class TransportValidateQueryAction extends TransportBroadcastOperationAct @Override protected String transportAction() { - return TransportActions.Admin.Indices.VALIDATE_QUERY; + return TransportActions.Admin.Indices.Validate.QUERY; } @Override diff --git a/src/main/java/org/elasticsearch/client/node/NodeClient.java b/src/main/java/org/elasticsearch/client/node/NodeClient.java index 8f52e14e8ab..aa733170ff6 100644 --- a/src/main/java/org/elasticsearch/client/node/NodeClient.java +++ b/src/main/java/org/elasticsearch/client/node/NodeClient.java @@ -21,6 +21,7 @@ package org.elasticsearch.client.node; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.TransportBulkAction; @@ -43,6 +44,7 @@ import org.elasticsearch.action.percolate.PercolateRequest; import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.percolate.TransportPercolateAction; import org.elasticsearch.action.search.*; +import org.elasticsearch.action.support.BaseAction; import org.elasticsearch.action.update.TransportUpdateAction; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; @@ -53,6 +55,8 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; +import java.util.Map; + /** * */ @@ -87,25 +91,21 @@ public class NodeClient extends AbstractClient implements InternalClient { private final TransportPercolateAction percolateAction; @Inject - public NodeClient(Settings settings, ThreadPool threadPool, NodeAdminClient admin, - TransportIndexAction indexAction, TransportUpdateAction updateAction, TransportDeleteAction deleteAction, TransportBulkAction bulkAction, - TransportDeleteByQueryAction deleteByQueryAction, TransportGetAction getAction, TransportMultiGetAction multiGetAction, TransportCountAction countAction, - TransportSearchAction searchAction, TransportSearchScrollAction searchScrollAction, - TransportMoreLikeThisAction moreLikeThisAction, TransportPercolateAction percolateAction) { + public NodeClient(Settings settings, ThreadPool threadPool, NodeAdminClient admin, Map actions) { this.threadPool = threadPool; this.admin = admin; - this.indexAction = indexAction; - this.updateAction = updateAction; - this.deleteAction = deleteAction; - this.bulkAction = bulkAction; - this.deleteByQueryAction = deleteByQueryAction; - this.getAction = getAction; - this.multiGetAction = multiGetAction; - this.countAction = countAction; - this.searchAction = searchAction; - this.searchScrollAction = searchScrollAction; - this.moreLikeThisAction = moreLikeThisAction; - this.percolateAction = percolateAction; + this.indexAction = (TransportIndexAction) actions.get(TransportActions.INDEX); + this.updateAction = (TransportUpdateAction) actions.get(TransportActions.UPDATE); + this.deleteAction = (TransportDeleteAction) actions.get(TransportActions.DELETE); + this.bulkAction = (TransportBulkAction) actions.get(TransportActions.BULK); + this.deleteByQueryAction = (TransportDeleteByQueryAction) actions.get(TransportActions.DELETE_BY_QUERY); + this.getAction = (TransportGetAction) actions.get(TransportActions.GET); + this.multiGetAction = (TransportMultiGetAction) actions.get(TransportActions.MULTI_GET); + this.countAction = (TransportCountAction) actions.get(TransportActions.COUNT); + this.searchAction = (TransportSearchAction) actions.get(TransportActions.SEARCH); + this.searchScrollAction = (TransportSearchScrollAction) actions.get(TransportActions.SEARCH_SCROLL); + this.moreLikeThisAction = (TransportMoreLikeThisAction) actions.get(TransportActions.MORE_LIKE_THIS); + this.percolateAction = (TransportPercolateAction) actions.get(TransportActions.PERCOLATE); } @Override diff --git a/src/main/java/org/elasticsearch/client/node/NodeClusterAdminClient.java b/src/main/java/org/elasticsearch/client/node/NodeClusterAdminClient.java index 81a2b14207b..d97b4b77c67 100644 --- a/src/main/java/org/elasticsearch/client/node/NodeClusterAdminClient.java +++ b/src/main/java/org/elasticsearch/client/node/NodeClusterAdminClient.java @@ -21,6 +21,7 @@ package org.elasticsearch.client.node; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction; @@ -54,12 +55,15 @@ import org.elasticsearch.action.admin.cluster.settings.TransportClusterUpdateSet import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.cluster.state.TransportClusterStateAction; +import org.elasticsearch.action.support.BaseAction; import org.elasticsearch.client.internal.InternalClusterAdminClient; import org.elasticsearch.client.support.AbstractClusterAdminClient; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; +import java.util.Map; + /** * */ @@ -90,22 +94,19 @@ public class NodeClusterAdminClient extends AbstractClusterAdminClient implement private final TransportNodesRestartAction nodesRestart; @Inject - public NodeClusterAdminClient(Settings settings, ThreadPool threadPool, - TransportClusterHealthAction clusterHealthAction, TransportClusterStateAction clusterStateAction, TransportClusterRerouteAction clusterRerouteAction, TransportClusterUpdateSettingsAction clusterUpdateSettingsAction, - TransportSinglePingAction singlePingAction, TransportBroadcastPingAction broadcastPingAction, TransportReplicationPingAction replicationPingAction, - TransportNodesInfoAction nodesInfoAction, TransportNodesShutdownAction nodesShutdown, TransportNodesRestartAction nodesRestart, TransportNodesStatsAction nodesStatsAction) { + public NodeClusterAdminClient(Settings settings, ThreadPool threadPool, Map actions) { this.threadPool = threadPool; - this.clusterRerouteAction = clusterRerouteAction; - this.clusterHealthAction = clusterHealthAction; - this.clusterStateAction = clusterStateAction; - this.clusterUpdateSettingsAction = clusterUpdateSettingsAction; - this.nodesInfoAction = nodesInfoAction; - this.nodesShutdown = nodesShutdown; - this.nodesRestart = nodesRestart; - this.singlePingAction = singlePingAction; - this.broadcastPingAction = broadcastPingAction; - this.replicationPingAction = replicationPingAction; - this.nodesStatsAction = nodesStatsAction; + this.clusterRerouteAction = (TransportClusterRerouteAction) actions.get(TransportActions.Admin.Cluster.REROUTE); + this.clusterHealthAction = (TransportClusterHealthAction) actions.get(TransportActions.Admin.Cluster.HEALTH); + this.clusterStateAction = (TransportClusterStateAction) actions.get(TransportActions.Admin.Cluster.STATE); + this.clusterUpdateSettingsAction = (TransportClusterUpdateSettingsAction) actions.get(TransportActions.Admin.Cluster.UPDATE_SETTINGS); + this.nodesInfoAction = (TransportNodesInfoAction) actions.get(TransportActions.Admin.Cluster.Node.INFO); + this.nodesStatsAction = (TransportNodesStatsAction) actions.get(TransportActions.Admin.Cluster.Node.STATS); + this.nodesShutdown = (TransportNodesShutdownAction) actions.get(TransportActions.Admin.Cluster.Node.SHUTDOWN); + this.nodesRestart = (TransportNodesRestartAction) actions.get(TransportActions.Admin.Cluster.Node.RESTART); + this.singlePingAction = (TransportSinglePingAction) actions.get(TransportActions.Admin.Cluster.Ping.SINGLE); + this.broadcastPingAction = (TransportBroadcastPingAction) actions.get(TransportActions.Admin.Cluster.Ping.BROADCAST); + this.replicationPingAction = (TransportReplicationPingAction) actions.get(TransportActions.Admin.Cluster.Ping.REPLICATION); } @Override diff --git a/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java b/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java index a0953a22a55..d318351c825 100644 --- a/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java +++ b/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java @@ -21,6 +21,7 @@ package org.elasticsearch.client.node; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction; @@ -84,12 +85,15 @@ import org.elasticsearch.action.admin.indices.template.put.TransportPutIndexTemp import org.elasticsearch.action.admin.indices.validate.query.TransportValidateQueryAction; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; +import org.elasticsearch.action.support.BaseAction; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.client.support.AbstractIndicesAdminClient; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; +import java.util.Map; + /** * */ @@ -140,36 +144,29 @@ public class NodeIndicesAdminClient extends AbstractIndicesAdminClient implement private final TransportValidateQueryAction validateQueryAction; @Inject - public NodeIndicesAdminClient(Settings settings, ThreadPool threadPool, TransportIndicesExistsAction indicesExistsAction, TransportIndicesStatsAction indicesStatsAction, TransportIndicesStatusAction indicesStatusAction, TransportIndicesSegmentsAction indicesSegmentsAction, - TransportCreateIndexAction createIndexAction, TransportDeleteIndexAction deleteIndexAction, - TransportCloseIndexAction closeIndexAction, TransportOpenIndexAction openIndexAction, - TransportRefreshAction refreshAction, TransportFlushAction flushAction, TransportOptimizeAction optimizeAction, - TransportPutMappingAction putMappingAction, TransportDeleteMappingAction deleteMappingAction, TransportGatewaySnapshotAction gatewaySnapshotAction, - TransportIndicesAliasesAction indicesAliasesAction, TransportClearIndicesCacheAction clearIndicesCacheAction, - TransportUpdateSettingsAction updateSettingsAction, TransportAnalyzeAction analyzeAction, - TransportPutIndexTemplateAction putIndexTemplateAction, TransportDeleteIndexTemplateAction deleteIndexTemplateAction, TransportValidateQueryAction validateQueryAction) { + public NodeIndicesAdminClient(Settings settings, ThreadPool threadPool, Map actions) { this.threadPool = threadPool; - this.indicesExistsAction = indicesExistsAction; - this.indicesStatsAction = indicesStatsAction; - this.indicesStatusAction = indicesStatusAction; - this.indicesSegmentsAction = indicesSegmentsAction; - this.createIndexAction = createIndexAction; - this.deleteIndexAction = deleteIndexAction; - this.closeIndexAction = closeIndexAction; - this.openIndexAction = openIndexAction; - this.refreshAction = refreshAction; - this.flushAction = flushAction; - this.optimizeAction = optimizeAction; - this.deleteMappingAction = deleteMappingAction; - this.putMappingAction = putMappingAction; - this.gatewaySnapshotAction = gatewaySnapshotAction; - this.indicesAliasesAction = indicesAliasesAction; - this.clearIndicesCacheAction = clearIndicesCacheAction; - this.updateSettingsAction = updateSettingsAction; - this.analyzeAction = analyzeAction; - this.putIndexTemplateAction = putIndexTemplateAction; - this.deleteIndexTemplateAction = deleteIndexTemplateAction; - this.validateQueryAction = validateQueryAction; + this.indicesExistsAction = (TransportIndicesExistsAction) actions.get(TransportActions.Admin.Indices.EXISTS); + this.indicesStatsAction = (TransportIndicesStatsAction) actions.get(TransportActions.Admin.Indices.STATS); + this.indicesStatusAction = (TransportIndicesStatusAction) actions.get(TransportActions.Admin.Indices.STATUS); + this.indicesSegmentsAction = (TransportIndicesSegmentsAction) actions.get(TransportActions.Admin.Indices.SEGMENTS); + this.createIndexAction = (TransportCreateIndexAction) actions.get(TransportActions.Admin.Indices.CREATE); + this.deleteIndexAction = (TransportDeleteIndexAction) actions.get(TransportActions.Admin.Indices.DELETE); + this.closeIndexAction = (TransportCloseIndexAction) actions.get(TransportActions.Admin.Indices.CLOSE); + this.openIndexAction = (TransportOpenIndexAction) actions.get(TransportActions.Admin.Indices.OPEN); + this.refreshAction = (TransportRefreshAction) actions.get(TransportActions.Admin.Indices.REFRESH); + this.flushAction = (TransportFlushAction) actions.get(TransportActions.Admin.Indices.FLUSH); + this.optimizeAction = (TransportOptimizeAction) actions.get(TransportActions.Admin.Indices.OPTIMIZE); + this.deleteMappingAction = (TransportDeleteMappingAction) actions.get(TransportActions.Admin.Indices.Mapping.DELETE); + this.putMappingAction = (TransportPutMappingAction) actions.get(TransportActions.Admin.Indices.Mapping.PUT); + this.gatewaySnapshotAction = (TransportGatewaySnapshotAction) actions.get(TransportActions.Admin.Indices.Gateway.SNAPSHOT); + this.indicesAliasesAction = (TransportIndicesAliasesAction) actions.get(TransportActions.Admin.Indices.ALIASES); + this.clearIndicesCacheAction = (TransportClearIndicesCacheAction) actions.get(TransportActions.Admin.Indices.Cache.CLEAR); + this.updateSettingsAction = (TransportUpdateSettingsAction) actions.get(TransportActions.Admin.Indices.UPDATE_SETTINGS); + this.analyzeAction = (TransportAnalyzeAction) actions.get(TransportActions.Admin.Indices.ANALYZE); + this.putIndexTemplateAction = (TransportPutIndexTemplateAction) actions.get(TransportActions.Admin.Indices.Template.PUT); + this.deleteIndexTemplateAction = (TransportDeleteIndexTemplateAction) actions.get(TransportActions.Admin.Indices.Template.DELETE); + this.validateQueryAction = (TransportValidateQueryAction) actions.get(TransportActions.Admin.Indices.Validate.QUERY); } @Override diff --git a/src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java b/src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java index 1db0d2f5053..67a84b71438 100644 --- a/src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java +++ b/src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java @@ -19,6 +19,8 @@ package org.elasticsearch.client.transport.action; +import com.google.common.collect.Maps; +import org.elasticsearch.action.TransportActions; import org.elasticsearch.client.transport.action.admin.cluster.health.ClientTransportClusterHealthAction; import org.elasticsearch.client.transport.action.admin.cluster.node.info.ClientTransportNodesInfoAction; import org.elasticsearch.client.transport.action.admin.cluster.node.restart.ClientTransportNodesRestartAction; @@ -61,60 +63,100 @@ import org.elasticsearch.client.transport.action.index.ClientTransportIndexActio import org.elasticsearch.client.transport.action.percolate.ClientTransportPercolateAction; import org.elasticsearch.client.transport.action.search.ClientTransportSearchAction; import org.elasticsearch.client.transport.action.search.ClientTransportSearchScrollAction; +import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; import org.elasticsearch.client.transport.action.update.ClientTransportUpdateAction; import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.inject.multibindings.MapBinder; + +import java.util.Map; /** * */ public class ClientTransportActionModule extends AbstractModule { + private final Map actions = Maps.newHashMap(); + + static class ActionEntry { + public final String name; + public final Class action; + public final Class[] supportActions; + + ActionEntry(String name, Class action, Class... supportActions) { + this.name = name; + this.action = action; + this.supportActions = supportActions; + } + } + + /** + * Registers a custom action under the provided action name, the actual action implementation, and + * any supported actions (bind as singletons). + * + * @param actionName The action name + * @param action The action itself + * @param supportActions Support actions. + */ + public void registerAction(String actionName, Class action, Class... supportActions) { + actions.put(actionName, new ActionEntry(actionName, action, supportActions)); + } + @Override protected void configure() { - bind(ClientTransportIndexAction.class).asEagerSingleton(); - bind(ClientTransportDeleteAction.class).asEagerSingleton(); - bind(ClientTransportDeleteByQueryAction.class).asEagerSingleton(); - bind(ClientTransportGetAction.class).asEagerSingleton(); - bind(ClientTransportMultiGetAction.class).asEagerSingleton(); - bind(ClientTransportCountAction.class).asEagerSingleton(); - bind(ClientTransportSearchAction.class).asEagerSingleton(); - bind(ClientTransportSearchScrollAction.class).asEagerSingleton(); - bind(ClientTransportBulkAction.class).asEagerSingleton(); - bind(ClientTransportPercolateAction.class).asEagerSingleton(); - bind(ClientTransportUpdateAction.class).asEagerSingleton(); + registerAction(TransportActions.INDEX, ClientTransportIndexAction.class); + registerAction(TransportActions.DELETE, ClientTransportDeleteAction.class); + registerAction(TransportActions.DELETE_BY_QUERY, ClientTransportDeleteByQueryAction.class); + registerAction(TransportActions.GET, ClientTransportGetAction.class); + registerAction(TransportActions.MULTI_GET, ClientTransportMultiGetAction.class); + registerAction(TransportActions.COUNT, ClientTransportCountAction.class); + registerAction(TransportActions.SEARCH, ClientTransportSearchAction.class); + registerAction(TransportActions.SEARCH_SCROLL, ClientTransportSearchScrollAction.class); + registerAction(TransportActions.BULK, ClientTransportBulkAction.class); + registerAction(TransportActions.PERCOLATE, ClientTransportPercolateAction.class); + registerAction(TransportActions.UPDATE, ClientTransportUpdateAction.class); - bind(ClientTransportIndicesExistsAction.class).asEagerSingleton(); - bind(ClientTransportIndicesStatsAction.class).asEagerSingleton(); - bind(ClientTransportIndicesStatusAction.class).asEagerSingleton(); - bind(ClientTransportIndicesSegmentsAction.class).asEagerSingleton(); - bind(ClientTransportRefreshAction.class).asEagerSingleton(); - bind(ClientTransportFlushAction.class).asEagerSingleton(); - bind(ClientTransportOptimizeAction.class).asEagerSingleton(); - bind(ClientTransportCreateIndexAction.class).asEagerSingleton(); - bind(ClientTransportDeleteIndexAction.class).asEagerSingleton(); - bind(ClientTransportCloseIndexAction.class).asEagerSingleton(); - bind(ClientTransportOpenIndexAction.class).asEagerSingleton(); - bind(ClientTransportPutMappingAction.class).asEagerSingleton(); - bind(ClientTransportDeleteMappingAction.class).asEagerSingleton(); - bind(ClientTransportGatewaySnapshotAction.class).asEagerSingleton(); - bind(ClientTransportIndicesAliasesAction.class).asEagerSingleton(); - bind(ClientTransportClearIndicesCacheAction.class).asEagerSingleton(); - bind(ClientTransportUpdateSettingsAction.class).asEagerSingleton(); - bind(ClientTransportAnalyzeAction.class).asEagerSingleton(); - bind(ClientTransportPutIndexTemplateAction.class).asEagerSingleton(); - bind(ClientTransportDeleteIndexTemplateAction.class).asEagerSingleton(); - bind(ClientTransportValidateQueryAction.class).asEagerSingleton(); + registerAction(TransportActions.Admin.Indices.EXISTS, ClientTransportIndicesExistsAction.class); + registerAction(TransportActions.Admin.Indices.STATS, ClientTransportIndicesStatsAction.class); + registerAction(TransportActions.Admin.Indices.STATUS, ClientTransportIndicesStatusAction.class); + registerAction(TransportActions.Admin.Indices.SEGMENTS, ClientTransportIndicesSegmentsAction.class); + registerAction(TransportActions.Admin.Indices.REFRESH, ClientTransportRefreshAction.class); + registerAction(TransportActions.Admin.Indices.FLUSH, ClientTransportFlushAction.class); + registerAction(TransportActions.Admin.Indices.OPTIMIZE, ClientTransportOptimizeAction.class); + registerAction(TransportActions.Admin.Indices.CREATE, ClientTransportCreateIndexAction.class); + registerAction(TransportActions.Admin.Indices.DELETE, ClientTransportDeleteIndexAction.class); + registerAction(TransportActions.Admin.Indices.CLOSE, ClientTransportCloseIndexAction.class); + registerAction(TransportActions.Admin.Indices.OPEN, ClientTransportOpenIndexAction.class); + registerAction(TransportActions.Admin.Indices.Mapping.PUT, ClientTransportPutMappingAction.class); + registerAction(TransportActions.Admin.Indices.Mapping.DELETE, ClientTransportDeleteMappingAction.class); + registerAction(TransportActions.Admin.Indices.Gateway.SNAPSHOT, ClientTransportGatewaySnapshotAction.class); + registerAction(TransportActions.Admin.Indices.ALIASES, ClientTransportIndicesAliasesAction.class); + registerAction(TransportActions.Admin.Indices.Cache.CLEAR, ClientTransportClearIndicesCacheAction.class); + registerAction(TransportActions.Admin.Indices.UPDATE_SETTINGS, ClientTransportUpdateSettingsAction.class); + registerAction(TransportActions.Admin.Indices.ANALYZE, ClientTransportAnalyzeAction.class); + registerAction(TransportActions.Admin.Indices.Template.PUT, ClientTransportPutIndexTemplateAction.class); + registerAction(TransportActions.Admin.Indices.Template.DELETE, ClientTransportDeleteIndexTemplateAction.class); + registerAction(TransportActions.Admin.Indices.Validate.QUERY, ClientTransportValidateQueryAction.class); - bind(ClientTransportNodesInfoAction.class).asEagerSingleton(); - bind(ClientTransportNodesStatsAction.class).asEagerSingleton(); - bind(ClientTransportNodesShutdownAction.class).asEagerSingleton(); - bind(ClientTransportNodesRestartAction.class).asEagerSingleton(); - bind(ClientTransportSinglePingAction.class).asEagerSingleton(); - bind(ClientTransportReplicationPingAction.class).asEagerSingleton(); - bind(ClientTransportBroadcastPingAction.class).asEagerSingleton(); - bind(ClientTransportClusterStateAction.class).asEagerSingleton(); - bind(ClientTransportClusterHealthAction.class).asEagerSingleton(); - bind(ClientTransportClusterUpdateSettingsAction.class).asEagerSingleton(); - bind(ClientTransportClusterRerouteAction.class).asEagerSingleton(); + registerAction(TransportActions.Admin.Cluster.Node.INFO, ClientTransportNodesInfoAction.class); + registerAction(TransportActions.Admin.Cluster.Node.STATS, ClientTransportNodesStatsAction.class); + registerAction(TransportActions.Admin.Cluster.Node.SHUTDOWN, ClientTransportNodesShutdownAction.class); + registerAction(TransportActions.Admin.Cluster.Node.RESTART, ClientTransportNodesRestartAction.class); + registerAction(TransportActions.Admin.Cluster.Ping.SINGLE, ClientTransportSinglePingAction.class); + registerAction(TransportActions.Admin.Cluster.Ping.REPLICATION, ClientTransportReplicationPingAction.class); + registerAction(TransportActions.Admin.Cluster.Ping.BROADCAST, ClientTransportBroadcastPingAction.class); + registerAction(TransportActions.Admin.Cluster.STATE, ClientTransportClusterStateAction.class); + registerAction(TransportActions.Admin.Cluster.HEALTH, ClientTransportClusterHealthAction.class); + registerAction(TransportActions.Admin.Cluster.UPDATE_SETTINGS, ClientTransportClusterUpdateSettingsAction.class); + registerAction(TransportActions.Admin.Cluster.REROUTE, ClientTransportClusterRerouteAction.class); + + MapBinder actionsBinder + = MapBinder.newMapBinder(binder(), String.class, BaseClientTransportAction.class); + + for (Map.Entry entry : actions.entrySet()) { + actionsBinder.addBinding(entry.getKey()).to(entry.getValue().action).asEagerSingleton(); + for (Class supportAction : entry.getValue().supportActions) { + bind(supportAction).asEagerSingleton(); + } + } } } diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/delete/ClientTransportDeleteIndexTemplateAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/delete/ClientTransportDeleteIndexTemplateAction.java index fe6d7d193b3..ab91878f9f0 100644 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/delete/ClientTransportDeleteIndexTemplateAction.java +++ b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/delete/ClientTransportDeleteIndexTemplateAction.java @@ -39,6 +39,6 @@ public class ClientTransportDeleteIndexTemplateAction extends BaseClientTranspor @Override protected String action() { - return TransportActions.Admin.Indices.DELETE_INDEX_TEMPLATE; + return TransportActions.Admin.Indices.Template.DELETE; } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/put/ClientTransportPutIndexTemplateAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/put/ClientTransportPutIndexTemplateAction.java index b8e7331cce6..38618792553 100644 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/put/ClientTransportPutIndexTemplateAction.java +++ b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/put/ClientTransportPutIndexTemplateAction.java @@ -39,6 +39,6 @@ public class ClientTransportPutIndexTemplateAction extends BaseClientTransportAc @Override protected String action() { - return TransportActions.Admin.Indices.PUT_INDEX_TEMPLATE; + return TransportActions.Admin.Indices.Template.PUT; } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/validate/query/ClientTransportValidateQueryAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/validate/query/ClientTransportValidateQueryAction.java index 1072b28c140..0612dfcbe87 100644 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/validate/query/ClientTransportValidateQueryAction.java +++ b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/validate/query/ClientTransportValidateQueryAction.java @@ -20,6 +20,6 @@ public class ClientTransportValidateQueryAction extends BaseClientTransportActio @Override protected String action() { - return TransportActions.Admin.Indices.VALIDATE_QUERY; + return TransportActions.Admin.Indices.Validate.QUERY; } } diff --git a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java index 526e1bbe3e5..8bfb59e35f0 100644 --- a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java +++ b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java @@ -22,6 +22,7 @@ package org.elasticsearch.client.transport.support; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.count.CountRequest; @@ -59,12 +60,15 @@ import org.elasticsearch.client.transport.action.mlt.ClientTransportMoreLikeThis import org.elasticsearch.client.transport.action.percolate.ClientTransportPercolateAction; import org.elasticsearch.client.transport.action.search.ClientTransportSearchAction; import org.elasticsearch.client.transport.action.search.ClientTransportSearchScrollAction; +import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; import org.elasticsearch.client.transport.action.update.ClientTransportUpdateAction; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; +import java.util.Map; + /** * */ @@ -103,26 +107,23 @@ public class InternalTransportClient extends AbstractClient implements InternalC @Inject public InternalTransportClient(Settings settings, ThreadPool threadPool, TransportClientNodesService nodesService, InternalTransportAdminClient adminClient, - ClientTransportIndexAction indexAction, ClientTransportUpdateAction updateAction, ClientTransportDeleteAction deleteAction, ClientTransportBulkAction bulkAction, ClientTransportGetAction getAction, ClientTransportMultiGetAction multiGetAction, - ClientTransportDeleteByQueryAction deleteByQueryAction, ClientTransportCountAction countAction, - ClientTransportSearchAction searchAction, ClientTransportSearchScrollAction searchScrollAction, - ClientTransportMoreLikeThisAction moreLikeThisAction, ClientTransportPercolateAction percolateAction) { + Map actions) { this.threadPool = threadPool; this.nodesService = nodesService; this.adminClient = adminClient; - this.indexAction = indexAction; - this.updateAction = updateAction; - this.deleteAction = deleteAction; - this.bulkAction = bulkAction; - this.getAction = getAction; - this.multiGetAction = multiGetAction; - this.deleteByQueryAction = deleteByQueryAction; - this.countAction = countAction; - this.searchAction = searchAction; - this.searchScrollAction = searchScrollAction; - this.moreLikeThisAction = moreLikeThisAction; - this.percolateAction = percolateAction; + this.indexAction = (ClientTransportIndexAction) actions.get(TransportActions.INDEX); + this.updateAction = (ClientTransportUpdateAction) actions.get(TransportActions.UPDATE); + this.deleteAction = (ClientTransportDeleteAction) actions.get(TransportActions.DELETE); + this.bulkAction = (ClientTransportBulkAction) actions.get(TransportActions.BULK); + this.getAction = (ClientTransportGetAction) actions.get(TransportActions.GET); + this.multiGetAction = (ClientTransportMultiGetAction) actions.get(TransportActions.MULTI_GET); + this.deleteByQueryAction = (ClientTransportDeleteByQueryAction) actions.get(TransportActions.DELETE_BY_QUERY); + this.countAction = (ClientTransportCountAction) actions.get(TransportActions.COUNT); + this.searchAction = (ClientTransportSearchAction) actions.get(TransportActions.SEARCH); + this.searchScrollAction = (ClientTransportSearchScrollAction) actions.get(TransportActions.SEARCH_SCROLL); + this.moreLikeThisAction = (ClientTransportMoreLikeThisAction) actions.get(TransportActions.MORE_LIKE_THIS); + this.percolateAction = (ClientTransportPercolateAction) actions.get(TransportActions.PERCOLATE); } @Override diff --git a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClusterAdminClient.java b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClusterAdminClient.java index 6b242ff99a0..f5c73322b07 100644 --- a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClusterAdminClient.java +++ b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClusterAdminClient.java @@ -22,6 +22,7 @@ package org.elasticsearch.client.transport.support; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; @@ -58,11 +59,14 @@ import org.elasticsearch.client.transport.action.admin.cluster.ping.single.Clien import org.elasticsearch.client.transport.action.admin.cluster.reroute.ClientTransportClusterRerouteAction; import org.elasticsearch.client.transport.action.admin.cluster.settings.ClientTransportClusterUpdateSettingsAction; import org.elasticsearch.client.transport.action.admin.cluster.state.ClientTransportClusterStateAction; +import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; +import java.util.Map; + /** * */ @@ -96,22 +100,20 @@ public class InternalTransportClusterAdminClient extends AbstractClusterAdminCli @Inject public InternalTransportClusterAdminClient(Settings settings, TransportClientNodesService nodesService, ThreadPool threadPool, - ClientTransportClusterHealthAction clusterHealthAction, ClientTransportClusterStateAction clusterStateAction, ClientTransportClusterRerouteAction clusterRerouteAction, ClientTransportClusterUpdateSettingsAction clusterUpdateSettingsAction, - ClientTransportSinglePingAction singlePingAction, ClientTransportReplicationPingAction replicationPingAction, ClientTransportBroadcastPingAction broadcastPingAction, - ClientTransportNodesInfoAction nodesInfoAction, ClientTransportNodesShutdownAction nodesShutdownAction, ClientTransportNodesRestartAction nodesRestartAction, ClientTransportNodesStatsAction nodesStatsAction) { + Map actions) { this.nodesService = nodesService; this.threadPool = threadPool; - this.clusterHealthAction = clusterHealthAction; - this.clusterRerouteAction = clusterRerouteAction; - this.clusterStateAction = clusterStateAction; - this.clusterUpdateSettingsAction = clusterUpdateSettingsAction; - this.nodesInfoAction = nodesInfoAction; - this.nodesShutdownAction = nodesShutdownAction; - this.nodesRestartAction = nodesRestartAction; - this.singlePingAction = singlePingAction; - this.replicationPingAction = replicationPingAction; - this.broadcastPingAction = broadcastPingAction; - this.nodesStatsAction = nodesStatsAction; + this.clusterHealthAction = (ClientTransportClusterHealthAction) actions.get(TransportActions.Admin.Cluster.HEALTH); + this.clusterRerouteAction = (ClientTransportClusterRerouteAction) actions.get(TransportActions.Admin.Cluster.REROUTE); + this.clusterStateAction = (ClientTransportClusterStateAction) actions.get(TransportActions.Admin.Cluster.STATE); + this.clusterUpdateSettingsAction = (ClientTransportClusterUpdateSettingsAction) actions.get(TransportActions.Admin.Cluster.UPDATE_SETTINGS); + this.nodesInfoAction = (ClientTransportNodesInfoAction) actions.get(TransportActions.Admin.Cluster.Node.INFO); + this.nodesShutdownAction = (ClientTransportNodesShutdownAction) actions.get(TransportActions.Admin.Cluster.Node.SHUTDOWN); + this.nodesRestartAction = (ClientTransportNodesRestartAction) actions.get(TransportActions.Admin.Cluster.Node.RESTART); + this.singlePingAction = (ClientTransportSinglePingAction) actions.get(TransportActions.Admin.Cluster.Ping.SINGLE); + this.replicationPingAction = (ClientTransportReplicationPingAction) actions.get(TransportActions.Admin.Cluster.Ping.REPLICATION); + this.broadcastPingAction = (ClientTransportBroadcastPingAction) actions.get(TransportActions.Admin.Cluster.Ping.BROADCAST); + this.nodesStatsAction = (ClientTransportNodesStatsAction) actions.get(TransportActions.Admin.Cluster.Node.STATS); } @Override diff --git a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportIndicesAdminClient.java b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportIndicesAdminClient.java index 12e7ce46a9e..4f6e5216c50 100644 --- a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportIndicesAdminClient.java +++ b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportIndicesAdminClient.java @@ -22,6 +22,7 @@ package org.elasticsearch.client.transport.support; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; @@ -88,11 +89,14 @@ import org.elasticsearch.client.transport.action.admin.indices.status.ClientTran import org.elasticsearch.client.transport.action.admin.indices.template.delete.ClientTransportDeleteIndexTemplateAction; import org.elasticsearch.client.transport.action.admin.indices.template.put.ClientTransportPutIndexTemplateAction; import org.elasticsearch.client.transport.action.admin.indices.validate.query.ClientTransportValidateQueryAction; +import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; +import java.util.Map; + /** * */ @@ -146,37 +150,30 @@ public class InternalTransportIndicesAdminClient extends AbstractIndicesAdminCli @Inject public InternalTransportIndicesAdminClient(Settings settings, TransportClientNodesService nodesService, ThreadPool threadPool, - ClientTransportIndicesExistsAction indicesExistsAction, ClientTransportIndicesStatusAction indicesStatusAction, ClientTransportIndicesStatsAction indicesStatsAction, ClientTransportIndicesSegmentsAction indicesSegmentsAction, - ClientTransportCreateIndexAction createIndexAction, ClientTransportDeleteIndexAction deleteIndexAction, - ClientTransportCloseIndexAction closeIndexAction, ClientTransportOpenIndexAction openIndexAction, - ClientTransportRefreshAction refreshAction, ClientTransportFlushAction flushAction, ClientTransportOptimizeAction optimizeAction, - ClientTransportPutMappingAction putMappingAction, ClientTransportDeleteMappingAction deleteMappingAction, ClientTransportGatewaySnapshotAction gatewaySnapshotAction, - ClientTransportIndicesAliasesAction indicesAliasesAction, ClientTransportClearIndicesCacheAction clearIndicesCacheAction, - ClientTransportUpdateSettingsAction updateSettingsAction, ClientTransportAnalyzeAction analyzeAction, ClientTransportValidateQueryAction validateQueryAction, - ClientTransportPutIndexTemplateAction putIndexTemplateAction, ClientTransportDeleteIndexTemplateAction deleteIndexTemplateAction) { + Map actions) { this.nodesService = nodesService; this.threadPool = threadPool; - this.indicesExistsAction = indicesExistsAction; - this.indicesStatsAction = indicesStatsAction; - this.indicesStatusAction = indicesStatusAction; - this.indicesSegmentsAction = indicesSegmentsAction; - this.createIndexAction = createIndexAction; - this.deleteIndexAction = deleteIndexAction; - this.closeIndexAction = closeIndexAction; - this.openIndexAction = openIndexAction; - this.refreshAction = refreshAction; - this.flushAction = flushAction; - this.optimizeAction = optimizeAction; - this.putMappingAction = putMappingAction; - this.deleteMappingAction = deleteMappingAction; - this.gatewaySnapshotAction = gatewaySnapshotAction; - this.indicesAliasesAction = indicesAliasesAction; - this.clearIndicesCacheAction = clearIndicesCacheAction; - this.updateSettingsAction = updateSettingsAction; - this.analyzeAction = analyzeAction; - this.putIndexTemplateAction = putIndexTemplateAction; - this.deleteIndexTemplateAction = deleteIndexTemplateAction; - this.validateQueryAction = validateQueryAction; + this.indicesExistsAction = (ClientTransportIndicesExistsAction) actions.get(TransportActions.Admin.Indices.EXISTS); + this.indicesStatsAction = (ClientTransportIndicesStatsAction) actions.get(TransportActions.Admin.Indices.STATS); + this.indicesStatusAction = (ClientTransportIndicesStatusAction) actions.get(TransportActions.Admin.Indices.STATUS); + this.indicesSegmentsAction = (ClientTransportIndicesSegmentsAction) actions.get(TransportActions.Admin.Indices.SEGMENTS); + this.createIndexAction = (ClientTransportCreateIndexAction) actions.get(TransportActions.Admin.Indices.CREATE); + this.deleteIndexAction = (ClientTransportDeleteIndexAction) actions.get(TransportActions.Admin.Indices.DELETE); + this.closeIndexAction = (ClientTransportCloseIndexAction) actions.get(TransportActions.Admin.Indices.CLOSE); + this.openIndexAction = (ClientTransportOpenIndexAction) actions.get(TransportActions.Admin.Indices.OPEN); + this.refreshAction = (ClientTransportRefreshAction) actions.get(TransportActions.Admin.Indices.REFRESH); + this.flushAction = (ClientTransportFlushAction) actions.get(TransportActions.Admin.Indices.FLUSH); + this.optimizeAction = (ClientTransportOptimizeAction) actions.get(TransportActions.Admin.Indices.OPTIMIZE); + this.putMappingAction = (ClientTransportPutMappingAction) actions.get(TransportActions.Admin.Indices.Mapping.PUT); + this.deleteMappingAction = (ClientTransportDeleteMappingAction) actions.get(TransportActions.Admin.Indices.Mapping.DELETE); + this.gatewaySnapshotAction = (ClientTransportGatewaySnapshotAction) actions.get(TransportActions.Admin.Indices.Gateway.SNAPSHOT); + this.indicesAliasesAction = (ClientTransportIndicesAliasesAction) actions.get(TransportActions.Admin.Indices.ALIASES); + this.clearIndicesCacheAction = (ClientTransportClearIndicesCacheAction) actions.get(TransportActions.Admin.Indices.Cache.CLEAR); + this.updateSettingsAction = (ClientTransportUpdateSettingsAction) actions.get(TransportActions.Admin.Indices.UPDATE_SETTINGS); + this.analyzeAction = (ClientTransportAnalyzeAction) actions.get(TransportActions.Admin.Indices.ANALYZE); + this.putIndexTemplateAction = (ClientTransportPutIndexTemplateAction) actions.get(TransportActions.Admin.Indices.Template.PUT); + this.deleteIndexTemplateAction = (ClientTransportDeleteIndexTemplateAction) actions.get(TransportActions.Admin.Indices.Template.DELETE); + this.validateQueryAction = (ClientTransportValidateQueryAction) actions.get(TransportActions.Admin.Indices.Validate.QUERY); } @Override From 4b21cf2993f4c71d984e6f07cc3aeb6f6e7f6ae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20Deve=CC=80ze?= Date: Sat, 14 Jan 2012 18:22:23 +0100 Subject: [PATCH 165/270] add percolation support to update action --- .../action/update/TransportUpdateAction.java | 6 +- .../action/update/UpdateRequest.java | 25 +++++++++ .../action/update/UpdateResponse.java | 56 +++++++++++++++++++ .../rest/action/update/RestUpdateAction.java | 9 +++ 4 files changed, 93 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java index ae9eea79b9f..8c1875429b9 100644 --- a/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java +++ b/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -208,20 +208,20 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio } } - // TODO percolate? - // TODO: external version type, does it make sense here? does not seem like it... if (operation == null || "index".equals(operation)) { IndexRequest indexRequest = Requests.indexRequest(request.index()).type(request.type()).id(request.id()).routing(routing).parent(parent) .source(source, sourceAndContent.v1()) .version(getResult.version()).replicationType(request.replicationType()).consistencyLevel(request.consistencyLevel()) - .timestamp(timestamp).ttl(ttl); + .timestamp(timestamp).ttl(ttl) + .percolate(request.percolate()); indexRequest.operationThreaded(false); indexAction.execute(indexRequest, new ActionListener() { @Override public void onResponse(IndexResponse response) { UpdateResponse update = new UpdateResponse(response.index(), response.type(), response.id(), response.version()); + update.matches(response.matches()); listener.onResponse(update); } diff --git a/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 05e39cee924..9934c6f43b5 100644 --- a/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -51,6 +51,8 @@ public class UpdateRequest extends InstanceShardOperationRequest { int retryOnConflict = 0; + private String percolate; + private ReplicationType replicationType = ReplicationType.DEFAULT; private WriteConsistencyLevel consistencyLevel = WriteConsistencyLevel.DEFAULT; @@ -239,6 +241,20 @@ public class UpdateRequest extends InstanceShardOperationRequest { return this.retryOnConflict; } + /** + * Causes the update request document to be percolated. The parameter is the percolate query + * to use to reduce the percolated queries that are going to run against this doc. Can be + * set to * to indicate that all percolate queries should be run. + */ + public UpdateRequest percolate(String percolate) { + this.percolate = percolate; + return this; + } + + public String percolate() { + return this.percolate; + } + /** * A timeout to wait if the index operation can't be performed immediately. Defaults to 1m. */ @@ -297,6 +313,9 @@ public class UpdateRequest extends InstanceShardOperationRequest { } scriptParams = in.readMap(); retryOnConflict = in.readVInt(); + if (in.readBoolean()) { + percolate = in.readUTF(); + } } @Override @@ -321,5 +340,11 @@ public class UpdateRequest extends InstanceShardOperationRequest { } out.writeMap(scriptParams); out.writeVInt(retryOnConflict); + if (percolate == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeUTF(percolate); + } } } diff --git a/src/main/java/org/elasticsearch/action/update/UpdateResponse.java b/src/main/java/org/elasticsearch/action/update/UpdateResponse.java index f4a41dd9d76..3cb6efdcbd8 100644 --- a/src/main/java/org/elasticsearch/action/update/UpdateResponse.java +++ b/src/main/java/org/elasticsearch/action/update/UpdateResponse.java @@ -19,11 +19,14 @@ package org.elasticsearch.action.update; +import com.google.common.collect.ImmutableList; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; /** */ @@ -37,6 +40,8 @@ public class UpdateResponse implements ActionResponse { private long version; + private List matches; + public UpdateResponse() { } @@ -104,12 +109,54 @@ public class UpdateResponse implements ActionResponse { return version(); } + /** + * Returns the percolate queries matches. null if no percolation was requested. + */ + public List matches() { + return this.matches; + } + + /** + * Returns the percolate queries matches. null if no percolation was requested. + */ + public List getMatches() { + return this.matches; + } + + /** + * Internal. + */ + public void matches(List matches) { + this.matches = matches; + } + @Override public void readFrom(StreamInput in) throws IOException { index = in.readUTF(); id = in.readUTF(); type = in.readUTF(); version = in.readLong(); + if (in.readBoolean()) { + int size = in.readVInt(); + if (size == 0) { + matches = ImmutableList.of(); + } else if (size == 1) { + matches = ImmutableList.of(in.readUTF()); + } else if (size == 2) { + matches = ImmutableList.of(in.readUTF(), in.readUTF()); + } else if (size == 3) { + matches = ImmutableList.of(in.readUTF(), in.readUTF(), in.readUTF()); + } else if (size == 4) { + matches = ImmutableList.of(in.readUTF(), in.readUTF(), in.readUTF(), in.readUTF()); + } else if (size == 5) { + matches = ImmutableList.of(in.readUTF(), in.readUTF(), in.readUTF(), in.readUTF(), in.readUTF()); + } else { + matches = new ArrayList(); + for (int i = 0; i < size; i++) { + matches.add(in.readUTF()); + } + } + } } @Override @@ -118,5 +165,14 @@ public class UpdateResponse implements ActionResponse { out.writeUTF(id); out.writeUTF(type); out.writeLong(version); + if (matches == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeVInt(matches.size()); + for (String match : matches) { + out.writeUTF(match); + } + } } } diff --git a/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java b/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java index e6bef6050b1..3e3eb366927 100644 --- a/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java +++ b/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java @@ -65,6 +65,7 @@ public class RestUpdateAction extends BaseRestHandler { if (consistencyLevel != null) { updateRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel)); } + updateRequest.percolate(request.param("percolate", null)); // we just send a response, no need to fork updateRequest.listenerThreaded(false); updateRequest.script(request.param("script")); @@ -114,6 +115,13 @@ public class RestUpdateAction extends BaseRestHandler { .field(Fields._TYPE, response.type()) .field(Fields._ID, response.id()) .field(Fields._VERSION, response.version()); + if (response.matches() != null) { + builder.startArray(Fields.MATCHES); + for (String match : response.matches()) { + builder.value(match); + } + builder.endArray(); + } builder.endObject(); RestStatus status = OK; if (response.version() == 1) { @@ -142,5 +150,6 @@ public class RestUpdateAction extends BaseRestHandler { static final XContentBuilderString _TYPE = new XContentBuilderString("_type"); static final XContentBuilderString _ID = new XContentBuilderString("_id"); static final XContentBuilderString _VERSION = new XContentBuilderString("_version"); + static final XContentBuilderString MATCHES = new XContentBuilderString("matches"); } } From 864c2ac96df63598ef32d60d217c7233c317f335 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 15 Jan 2012 01:08:39 +0200 Subject: [PATCH 166/270] remove Action interface and rename BaseAction to TransportAction (all transport level actions should extend it) --- .../java/org/elasticsearch/action/Action.java | 32 ------------------- .../action/TransportActionModule.java | 12 +++---- .../action/bulk/TransportBulkAction.java | 4 +-- .../action/get/TransportMultiGetAction.java | 4 +-- .../mlt/TransportMoreLikeThisAction.java | 6 ++-- .../action/search/TransportSearchAction.java | 4 +-- .../search/TransportSearchScrollAction.java | 4 +-- .../type/TransportSearchTypeAction.java | 4 +-- .../{BaseAction.java => TransportAction.java} | 6 ++-- .../TransportBroadcastOperationAction.java | 4 +-- .../TransportMasterNodeOperationAction.java | 6 ++-- .../nodes/TransportNodesOperationAction.java | 4 +-- ...nsportIndexReplicationOperationAction.java | 4 +-- ...portIndicesReplicationOperationAction.java | 4 +-- ...nsportShardReplicationOperationAction.java | 4 +-- .../TransportSingleCustomOperationAction.java | 4 +-- ...ransportInstanceSingleOperationAction.java | 4 +-- .../TransportShardSingleOperationAction.java | 4 +-- .../elasticsearch/client/node/NodeClient.java | 4 +-- .../client/node/NodeClusterAdminClient.java | 4 +-- .../client/node/NodeIndicesAdminClient.java | 4 +-- 21 files changed, 44 insertions(+), 82 deletions(-) delete mode 100644 src/main/java/org/elasticsearch/action/Action.java rename src/main/java/org/elasticsearch/action/support/{BaseAction.java => TransportAction.java} (93%) diff --git a/src/main/java/org/elasticsearch/action/Action.java b/src/main/java/org/elasticsearch/action/Action.java deleted file mode 100644 index bc3ca22cfbb..00000000000 --- a/src/main/java/org/elasticsearch/action/Action.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action; - -import org.elasticsearch.ElasticSearchException; - -/** - * - */ -public interface Action { - - ActionFuture execute(Request request) throws ElasticSearchException; - - void execute(Request request, ActionListener listener); -} diff --git a/src/main/java/org/elasticsearch/action/TransportActionModule.java b/src/main/java/org/elasticsearch/action/TransportActionModule.java index 5a6765df6bc..4f9d308930f 100644 --- a/src/main/java/org/elasticsearch/action/TransportActionModule.java +++ b/src/main/java/org/elasticsearch/action/TransportActionModule.java @@ -72,7 +72,7 @@ import org.elasticsearch.action.percolate.TransportPercolateAction; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.search.TransportSearchScrollAction; import org.elasticsearch.action.search.type.*; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.update.TransportUpdateAction; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.multibindings.MapBinder; @@ -88,10 +88,10 @@ public class TransportActionModule extends AbstractModule { static class ActionEntry { public final String name; - public final Class action; + public final Class action; public final Class[] supportActions; - ActionEntry(String name, Class action, Class... supportActions) { + ActionEntry(String name, Class action, Class... supportActions) { this.name = name; this.action = action; this.supportActions = supportActions; @@ -111,7 +111,7 @@ public class TransportActionModule extends AbstractModule { * @param action The action itself * @param supportActions Support actions. */ - public void registerAction(String actionName, Class action, Class... supportActions) { + public void registerAction(String actionName, Class action, Class... supportActions) { actions.put(actionName, new ActionEntry(actionName, action, supportActions)); } @@ -183,8 +183,8 @@ public class TransportActionModule extends AbstractModule { registerAction(TransportActions.MORE_LIKE_THIS, TransportMoreLikeThisAction.class); registerAction(TransportActions.PERCOLATE, TransportPercolateAction.class); - MapBinder actionsBinder - = MapBinder.newMapBinder(binder(), String.class, BaseAction.class); + MapBinder actionsBinder + = MapBinder.newMapBinder(binder(), String.class, TransportAction.class); for (Map.Entry entry : actions.entrySet()) { actionsBinder.addBinding(entry.getKey()).to(entry.getValue().action).asEagerSingleton(); diff --git a/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 35dc277200d..0331b98fa2a 100644 --- a/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -31,7 +31,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -57,7 +57,7 @@ import java.util.concurrent.atomic.AtomicInteger; /** * */ -public class TransportBulkAction extends BaseAction { +public class TransportBulkAction extends TransportAction { private final boolean autoCreateIndex; diff --git a/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java b/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java index 3819bdb69a5..bf0a40ec070 100644 --- a/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java +++ b/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java @@ -22,7 +22,7 @@ package org.elasticsearch.action.get; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -38,7 +38,7 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -public class TransportMultiGetAction extends BaseAction { +public class TransportMultiGetAction extends TransportAction { private final ClusterService clusterService; diff --git a/src/main/java/org/elasticsearch/action/mlt/TransportMoreLikeThisAction.java b/src/main/java/org/elasticsearch/action/mlt/TransportMoreLikeThisAction.java index de775d2cc2b..68d51e964b9 100644 --- a/src/main/java/org/elasticsearch/action/mlt/TransportMoreLikeThisAction.java +++ b/src/main/java/org/elasticsearch/action/mlt/TransportMoreLikeThisAction.java @@ -30,7 +30,7 @@ import org.elasticsearch.action.get.TransportGetAction; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.TransportSearchAction; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.inject.Inject; @@ -58,10 +58,8 @@ import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; /** * The more like this action. - * - * */ -public class TransportMoreLikeThisAction extends BaseAction { +public class TransportMoreLikeThisAction extends TransportAction { private final TransportSearchAction searchAction; diff --git a/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 09c5ad45fda..f384c49e790 100644 --- a/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -22,7 +22,7 @@ package org.elasticsearch.action.search; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.search.type.*; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.inject.Inject; @@ -41,7 +41,7 @@ import static org.elasticsearch.action.search.SearchType.*; /** * */ -public class TransportSearchAction extends BaseAction { +public class TransportSearchAction extends TransportAction { private final ClusterService clusterService; diff --git a/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java b/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java index 71f98419fde..dcc208bba01 100644 --- a/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java +++ b/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java @@ -26,7 +26,7 @@ import org.elasticsearch.action.search.type.ParsedScrollId; import org.elasticsearch.action.search.type.TransportSearchScrollQueryAndFetchAction; import org.elasticsearch.action.search.type.TransportSearchScrollQueryThenFetchAction; import org.elasticsearch.action.search.type.TransportSearchScrollScanAction; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; @@ -40,7 +40,7 @@ import static org.elasticsearch.action.search.type.TransportSearchHelper.parseSc /** * */ -public class TransportSearchScrollAction extends BaseAction { +public class TransportSearchScrollAction extends TransportAction { private final TransportSearchScrollQueryThenFetchAction queryThenFetchAction; diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java index fd610cf1edf..7ab9f545640 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java @@ -21,7 +21,7 @@ package org.elasticsearch.action.search.type; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.*; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -54,7 +54,7 @@ import static org.elasticsearch.action.search.type.TransportSearchHelper.interna /** * */ -public abstract class TransportSearchTypeAction extends BaseAction { +public abstract class TransportSearchTypeAction extends TransportAction { protected final ClusterService clusterService; diff --git a/src/main/java/org/elasticsearch/action/support/BaseAction.java b/src/main/java/org/elasticsearch/action/support/TransportAction.java similarity index 93% rename from src/main/java/org/elasticsearch/action/support/BaseAction.java rename to src/main/java/org/elasticsearch/action/support/TransportAction.java index 1247d924006..8d604dd5a9c 100644 --- a/src/main/java/org/elasticsearch/action/support/BaseAction.java +++ b/src/main/java/org/elasticsearch/action/support/TransportAction.java @@ -30,16 +30,15 @@ import static org.elasticsearch.action.support.PlainActionFuture.newFuture; /** * */ -public abstract class BaseAction extends AbstractComponent implements Action { +public abstract class TransportAction extends AbstractComponent { protected final ThreadPool threadPool; - protected BaseAction(Settings settings, ThreadPool threadPool) { + protected TransportAction(Settings settings, ThreadPool threadPool) { super(settings); this.threadPool = threadPool; } - @Override public ActionFuture execute(Request request) throws ElasticSearchException { PlainActionFuture future = newFuture(); // since we don't have a listener, and we release a possible lock with the future @@ -49,7 +48,6 @@ public abstract class BaseAction listener) { if (request.listenerThreaded()) { listener = new ThreadedActionListener(threadPool, listener); diff --git a/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastOperationAction.java b/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastOperationAction.java index 32e45a13816..ed3e8d547a9 100644 --- a/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastOperationAction.java @@ -21,7 +21,7 @@ package org.elasticsearch.action.support.broadcast; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; @@ -42,7 +42,7 @@ import java.util.concurrent.atomic.AtomicReferenceArray; * */ public abstract class TransportBroadcastOperationAction - extends BaseAction { + extends TransportAction { protected final ClusterService clusterService; diff --git a/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeOperationAction.java b/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeOperationAction.java index 8e5eee0ab65..a058787c256 100644 --- a/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeOperationAction.java @@ -22,7 +22,7 @@ package org.elasticsearch.action.support.master; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -38,10 +38,8 @@ import org.elasticsearch.transport.*; /** * A base class for operations that needs to be performed on the master node. - * - * */ -public abstract class TransportMasterNodeOperationAction extends BaseAction { +public abstract class TransportMasterNodeOperationAction extends TransportAction { protected final TransportService transportService; diff --git a/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesOperationAction.java b/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesOperationAction.java index ce4e48586a0..7a4f4fbf9f9 100644 --- a/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesOperationAction.java @@ -23,7 +23,7 @@ import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.NoSuchNodeException; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -40,7 +40,7 @@ import java.util.concurrent.atomic.AtomicReferenceArray; /** * */ -public abstract class TransportNodesOperationAction extends BaseAction { +public abstract class TransportNodesOperationAction extends TransportAction { protected final ClusterName clusterName; diff --git a/src/main/java/org/elasticsearch/action/support/replication/TransportIndexReplicationOperationAction.java b/src/main/java/org/elasticsearch/action/support/replication/TransportIndexReplicationOperationAction.java index 43a1536d398..e26eecb4a44 100644 --- a/src/main/java/org/elasticsearch/action/support/replication/TransportIndexReplicationOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/replication/TransportIndexReplicationOperationAction.java @@ -23,7 +23,7 @@ import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; @@ -43,7 +43,7 @@ import java.util.concurrent.atomic.AtomicReferenceArray; * */ public abstract class TransportIndexReplicationOperationAction - extends BaseAction { + extends TransportAction { protected final ClusterService clusterService; diff --git a/src/main/java/org/elasticsearch/action/support/replication/TransportIndicesReplicationOperationAction.java b/src/main/java/org/elasticsearch/action/support/replication/TransportIndicesReplicationOperationAction.java index 251d8ce9073..4ffcf480c35 100644 --- a/src/main/java/org/elasticsearch/action/support/replication/TransportIndicesReplicationOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/replication/TransportIndicesReplicationOperationAction.java @@ -22,7 +22,7 @@ package org.elasticsearch.action.support.replication; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; @@ -42,7 +42,7 @@ import java.util.concurrent.atomic.AtomicReferenceArray; */ public abstract class TransportIndicesReplicationOperationAction - extends BaseAction { + extends TransportAction { protected final ClusterService clusterService; diff --git a/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java b/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java index 95e3b02f969..14fb006d550 100644 --- a/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java @@ -22,7 +22,7 @@ package org.elasticsearch.action.support.replication; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.*; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -59,7 +59,7 @@ import static org.elasticsearch.ExceptionsHelper.detailedMessage; /** */ -public abstract class TransportShardReplicationOperationAction extends BaseAction { +public abstract class TransportShardReplicationOperationAction extends TransportAction { protected final TransportService transportService; diff --git a/src/main/java/org/elasticsearch/action/support/single/custom/TransportSingleCustomOperationAction.java b/src/main/java/org/elasticsearch/action/support/single/custom/TransportSingleCustomOperationAction.java index 97957ea3432..edb47af79b8 100644 --- a/src/main/java/org/elasticsearch/action/support/single/custom/TransportSingleCustomOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/single/custom/TransportSingleCustomOperationAction.java @@ -23,7 +23,7 @@ import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.NoShardAvailableActionException; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; @@ -43,7 +43,7 @@ import java.io.IOException; /** * */ -public abstract class TransportSingleCustomOperationAction extends BaseAction { +public abstract class TransportSingleCustomOperationAction extends TransportAction { protected final ClusterService clusterService; diff --git a/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java index 1ead9d802d5..b257fc9970c 100644 --- a/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java @@ -23,7 +23,7 @@ import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.UnavailableShardsException; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -46,7 +46,7 @@ import java.util.concurrent.atomic.AtomicBoolean; /** * */ -public abstract class TransportInstanceSingleOperationAction extends BaseAction { +public abstract class TransportInstanceSingleOperationAction extends TransportAction { protected final ClusterService clusterService; diff --git a/src/main/java/org/elasticsearch/action/support/single/shard/TransportShardSingleOperationAction.java b/src/main/java/org/elasticsearch/action/support/single/shard/TransportShardSingleOperationAction.java index 8e54c55ce95..b10fcd1c4aa 100644 --- a/src/main/java/org/elasticsearch/action/support/single/shard/TransportShardSingleOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/single/shard/TransportShardSingleOperationAction.java @@ -23,7 +23,7 @@ import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.NoShardAvailableActionException; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; @@ -44,7 +44,7 @@ import java.io.IOException; /** * */ -public abstract class TransportShardSingleOperationAction extends BaseAction { +public abstract class TransportShardSingleOperationAction extends TransportAction { protected final ClusterService clusterService; diff --git a/src/main/java/org/elasticsearch/client/node/NodeClient.java b/src/main/java/org/elasticsearch/client/node/NodeClient.java index aa733170ff6..2de1400df69 100644 --- a/src/main/java/org/elasticsearch/client/node/NodeClient.java +++ b/src/main/java/org/elasticsearch/client/node/NodeClient.java @@ -44,7 +44,7 @@ import org.elasticsearch.action.percolate.PercolateRequest; import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.percolate.TransportPercolateAction; import org.elasticsearch.action.search.*; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.update.TransportUpdateAction; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; @@ -91,7 +91,7 @@ public class NodeClient extends AbstractClient implements InternalClient { private final TransportPercolateAction percolateAction; @Inject - public NodeClient(Settings settings, ThreadPool threadPool, NodeAdminClient admin, Map actions) { + public NodeClient(Settings settings, ThreadPool threadPool, NodeAdminClient admin, Map actions) { this.threadPool = threadPool; this.admin = admin; this.indexAction = (TransportIndexAction) actions.get(TransportActions.INDEX); diff --git a/src/main/java/org/elasticsearch/client/node/NodeClusterAdminClient.java b/src/main/java/org/elasticsearch/client/node/NodeClusterAdminClient.java index d97b4b77c67..d0f81e75340 100644 --- a/src/main/java/org/elasticsearch/client/node/NodeClusterAdminClient.java +++ b/src/main/java/org/elasticsearch/client/node/NodeClusterAdminClient.java @@ -55,7 +55,7 @@ import org.elasticsearch.action.admin.cluster.settings.TransportClusterUpdateSet import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.cluster.state.TransportClusterStateAction; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.internal.InternalClusterAdminClient; import org.elasticsearch.client.support.AbstractClusterAdminClient; import org.elasticsearch.common.inject.Inject; @@ -94,7 +94,7 @@ public class NodeClusterAdminClient extends AbstractClusterAdminClient implement private final TransportNodesRestartAction nodesRestart; @Inject - public NodeClusterAdminClient(Settings settings, ThreadPool threadPool, Map actions) { + public NodeClusterAdminClient(Settings settings, ThreadPool threadPool, Map actions) { this.threadPool = threadPool; this.clusterRerouteAction = (TransportClusterRerouteAction) actions.get(TransportActions.Admin.Cluster.REROUTE); this.clusterHealthAction = (TransportClusterHealthAction) actions.get(TransportActions.Admin.Cluster.HEALTH); diff --git a/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java b/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java index d318351c825..c69ea9b9f48 100644 --- a/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java +++ b/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java @@ -85,7 +85,7 @@ import org.elasticsearch.action.admin.indices.template.put.TransportPutIndexTemp import org.elasticsearch.action.admin.indices.validate.query.TransportValidateQueryAction; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; -import org.elasticsearch.action.support.BaseAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.client.support.AbstractIndicesAdminClient; import org.elasticsearch.common.inject.Inject; @@ -144,7 +144,7 @@ public class NodeIndicesAdminClient extends AbstractIndicesAdminClient implement private final TransportValidateQueryAction validateQueryAction; @Inject - public NodeIndicesAdminClient(Settings settings, ThreadPool threadPool, Map actions) { + public NodeIndicesAdminClient(Settings settings, ThreadPool threadPool, Map actions) { this.threadPool = threadPool; this.indicesExistsAction = (TransportIndicesExistsAction) actions.get(TransportActions.Admin.Indices.EXISTS); this.indicesStatsAction = (TransportIndicesStatsAction) actions.get(TransportActions.Admin.Indices.STATS); From 6ff15b928c03079fe92cc061607b3e499d4d35fb Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 15 Jan 2012 12:09:18 +0200 Subject: [PATCH 167/270] rename Actions to ValidateActions --- .../action/{Actions.java => ValidateActions.java} | 2 +- .../cluster/settings/ClusterUpdateSettingsRequest.java | 2 +- .../admin/indices/alias/IndicesAliasesRequest.java | 4 +--- .../action/admin/indices/analyze/AnalyzeRequest.java | 2 +- .../action/admin/indices/close/CloseIndexRequest.java | 4 +--- .../action/admin/indices/create/CreateIndexRequest.java | 3 +-- .../action/admin/indices/delete/DeleteIndexRequest.java | 4 +--- .../admin/indices/exists/IndicesExistsRequest.java | 2 +- .../indices/mapping/delete/DeleteMappingRequest.java | 2 +- .../admin/indices/mapping/put/PutMappingRequest.java | 3 +-- .../action/admin/indices/open/OpenIndexRequest.java | 4 +--- .../admin/indices/settings/UpdateSettingsRequest.java | 2 +- .../template/delete/DeleteIndexTemplateRequest.java | 4 +--- .../indices/template/put/PutIndexTemplateRequest.java | 2 +- .../java/org/elasticsearch/action/bulk/BulkRequest.java | 3 +-- .../org/elasticsearch/action/delete/DeleteRequest.java | 3 +-- .../action/delete/index/ShardDeleteRequest.java | 4 +--- .../action/deletebyquery/DeleteByQueryRequest.java | 2 +- .../action/deletebyquery/IndexDeleteByQueryRequest.java | 2 +- .../action/deletebyquery/ShardDeleteByQueryRequest.java | 2 +- .../java/org/elasticsearch/action/get/GetRequest.java | 7 +++---- .../org/elasticsearch/action/get/MultiGetRequest.java | 8 ++++---- .../org/elasticsearch/action/index/IndexRequest.java | 2 +- .../elasticsearch/action/mlt/MoreLikeThisRequest.java | 9 ++++----- .../elasticsearch/action/percolate/PercolateRequest.java | 2 +- .../org/elasticsearch/action/search/SearchRequest.java | 2 +- .../elasticsearch/action/search/SearchScrollRequest.java | 2 +- .../replication/IndexReplicationOperationRequest.java | 2 +- .../replication/ShardReplicationOperationRequest.java | 2 +- .../single/instance/InstanceShardOperationRequest.java | 4 ++-- .../single/shard/SingleShardOperationRequest.java | 4 ++-- .../org/elasticsearch/action/update/UpdateRequest.java | 2 +- 32 files changed, 42 insertions(+), 60 deletions(-) rename src/main/java/org/elasticsearch/action/{Actions.java => ValidateActions.java} (97%) diff --git a/src/main/java/org/elasticsearch/action/Actions.java b/src/main/java/org/elasticsearch/action/ValidateActions.java similarity index 97% rename from src/main/java/org/elasticsearch/action/Actions.java rename to src/main/java/org/elasticsearch/action/ValidateActions.java index 4aaa912eb80..a5978eafcbc 100644 --- a/src/main/java/org/elasticsearch/action/Actions.java +++ b/src/main/java/org/elasticsearch/action/ValidateActions.java @@ -22,7 +22,7 @@ package org.elasticsearch.action; /** * */ -public class Actions { +public class ValidateActions { public static ActionRequestValidationException addValidationError(String error, ActionRequestValidationException validationException) { if (validationException == null) { diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java b/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java index 79e6dc33575..3108139dc4c 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java @@ -33,7 +33,7 @@ import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.util.Map; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.common.settings.ImmutableSettings.readSettingsFromStream; import static org.elasticsearch.common.settings.ImmutableSettings.writeSettingsToStream; diff --git a/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java index ff2da4df521..3a826772b8b 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java @@ -37,14 +37,12 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.cluster.metadata.AliasAction.readAliasAction; import static org.elasticsearch.common.unit.TimeValue.readTimeValue; /** * A request to add/remove aliases for one or more indices. - * - * */ public class IndicesAliasesRequest extends MasterNodeOperationRequest { diff --git a/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java index 78200018bc1..23954b3f3a9 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java @@ -27,7 +27,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; /** * A request to analyze a text associated with a specific index. Allow to provide diff --git a/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java index 66c172995e8..3726228ab77 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java @@ -27,14 +27,12 @@ import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.common.unit.TimeValue.readTimeValue; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; /** * A request to close an index. - * - * */ public class CloseIndexRequest extends MasterNodeOperationRequest { diff --git a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 4172c79a418..1ba3cebd143 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -38,7 +38,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import static com.google.common.collect.Maps.newHashMap; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.common.settings.ImmutableSettings.readSettingsFromStream; import static org.elasticsearch.common.settings.ImmutableSettings.writeSettingsToStream; @@ -49,7 +49,6 @@ import static org.elasticsearch.common.unit.TimeValue.readTimeValue; *

    *

    The index created can optionally be created with {@link #settings(org.elasticsearch.common.settings.Settings)}. * - * * @see org.elasticsearch.client.IndicesAdminClient#create(CreateIndexRequest) * @see org.elasticsearch.client.Requests#createIndexRequest(String) * @see CreateIndexResponse diff --git a/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java index 276ac28ca9d..bf3ab045709 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java @@ -27,14 +27,12 @@ import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.common.unit.TimeValue.readTimeValue; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; /** * A request to delete an index. Best created with {@link org.elasticsearch.client.Requests#deleteIndexRequest(String)}. - * - * */ public class DeleteIndexRequest extends MasterNodeOperationRequest { diff --git a/src/main/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsRequest.java index ddd1e641b17..1b6492dd209 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsRequest.java @@ -26,7 +26,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; public class IndicesExistsRequest extends MasterNodeOperationRequest { diff --git a/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/DeleteMappingRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/DeleteMappingRequest.java index 854c55359e5..63926d0b788 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/DeleteMappingRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/DeleteMappingRequest.java @@ -26,7 +26,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; /** * diff --git a/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java index 400427b0b83..14c5f4daa8c 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java @@ -35,7 +35,7 @@ import java.io.IOException; import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.common.unit.TimeValue.readTimeValue; /** @@ -46,7 +46,6 @@ import static org.elasticsearch.common.unit.TimeValue.readTimeValue; * that can't be merged are detected, the request will be rejected unless the {@link #ignoreConflicts(boolean)} * is set. In such a case, the duplicate mappings will be rejected. * - * * @see org.elasticsearch.client.Requests#putMappingRequest(String...) * @see org.elasticsearch.client.IndicesAdminClient#putMapping(PutMappingRequest) * @see PutMappingResponse diff --git a/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequest.java index d9bdd78710e..c64bd074c90 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequest.java @@ -27,14 +27,12 @@ import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.common.unit.TimeValue.readTimeValue; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; /** * A request to open an index. - * - * */ public class OpenIndexRequest extends MasterNodeOperationRequest { diff --git a/src/main/java/org/elasticsearch/action/admin/indices/settings/UpdateSettingsRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/settings/UpdateSettingsRequest.java index 149fcf4d446..21c5879c53a 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/settings/UpdateSettingsRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/settings/UpdateSettingsRequest.java @@ -33,7 +33,7 @@ import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.util.Map; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.common.settings.ImmutableSettings.readSettingsFromStream; import static org.elasticsearch.common.settings.ImmutableSettings.writeSettingsToStream; diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java index 07d0e18b33e..1424bfc2539 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java @@ -27,14 +27,12 @@ import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.common.unit.TimeValue.readTimeValue; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; /** * A request to delete an index template. - * - * */ public class DeleteIndexTemplateRequest extends MasterNodeOperationRequest { diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index b3934931510..90eeaca27ae 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -38,7 +38,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import static com.google.common.collect.Maps.newHashMap; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.common.settings.ImmutableSettings.readSettingsFromStream; import static org.elasticsearch.common.settings.ImmutableSettings.writeSettingsToStream; diff --git a/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index a5a49a7a83e..3c7472ea8ab 100644 --- a/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -38,13 +38,12 @@ import org.elasticsearch.index.VersionType; import java.io.IOException; import java.util.List; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; /** * A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes * it in a single batch. * - * * @see org.elasticsearch.client.Client#bulk(BulkRequest) */ public class BulkRequest implements ActionRequest { diff --git a/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java b/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java index 0625a7671bf..74a4e8719ee 100644 --- a/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java +++ b/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java @@ -32,7 +32,7 @@ import org.elasticsearch.index.VersionType; import java.io.IOException; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; /** * A request to delete a document from an index based on its type and id. Best created using @@ -41,7 +41,6 @@ import static org.elasticsearch.action.Actions.addValidationError; *

    The operation requires the {@link #index()}, {@link #type(String)} and {@link #id(String)} to * be set. * - * * @see DeleteResponse * @see org.elasticsearch.client.Client#delete(DeleteRequest) * @see org.elasticsearch.client.Requests#deleteRequest(String) diff --git a/src/main/java/org/elasticsearch/action/delete/index/ShardDeleteRequest.java b/src/main/java/org/elasticsearch/action/delete/index/ShardDeleteRequest.java index 1a98942b571..7bfa78cae64 100644 --- a/src/main/java/org/elasticsearch/action/delete/index/ShardDeleteRequest.java +++ b/src/main/java/org/elasticsearch/action/delete/index/ShardDeleteRequest.java @@ -26,12 +26,10 @@ import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; /** * Delete by query request to execute on a specific shard. - * - * */ public class ShardDeleteRequest extends ShardReplicationOperationRequest { diff --git a/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java b/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java index 25f4d917214..53010e4cfe0 100644 --- a/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java +++ b/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java @@ -40,7 +40,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Map; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; /** * A request to delete all documents that matching a specific query. Best created with diff --git a/src/main/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryRequest.java b/src/main/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryRequest.java index fd86bbb52c8..2a618454ef5 100644 --- a/src/main/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryRequest.java +++ b/src/main/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryRequest.java @@ -32,7 +32,7 @@ import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; import java.util.Set; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; /** * Delete by query request to execute on a specific index. diff --git a/src/main/java/org/elasticsearch/action/deletebyquery/ShardDeleteByQueryRequest.java b/src/main/java/org/elasticsearch/action/deletebyquery/ShardDeleteByQueryRequest.java index c1a68772d01..bba453bce0f 100644 --- a/src/main/java/org/elasticsearch/action/deletebyquery/ShardDeleteByQueryRequest.java +++ b/src/main/java/org/elasticsearch/action/deletebyquery/ShardDeleteByQueryRequest.java @@ -33,7 +33,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Set; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; /** * Delete by query request to execute on a specific shard. diff --git a/src/main/java/org/elasticsearch/action/get/GetRequest.java b/src/main/java/org/elasticsearch/action/get/GetRequest.java index ac02194aa58..596130ddf87 100644 --- a/src/main/java/org/elasticsearch/action/get/GetRequest.java +++ b/src/main/java/org/elasticsearch/action/get/GetRequest.java @@ -20,7 +20,7 @@ package org.elasticsearch.action.get; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.Actions; +import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.support.single.shard.SingleShardOperationRequest; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Required; @@ -36,7 +36,6 @@ import java.io.IOException; *

    The operation requires the {@link #index()}, {@link #type(String)} and {@link #id(String)} * to be set. * - * * @see org.elasticsearch.action.get.GetResponse * @see org.elasticsearch.client.Requests#getRequest(String) * @see org.elasticsearch.client.Client#get(GetRequest) @@ -84,10 +83,10 @@ public class GetRequest extends SingleShardOperationRequest { public ActionRequestValidationException validate() { ActionRequestValidationException validationException = super.validate(); if (type == null) { - validationException = Actions.addValidationError("type is missing", validationException); + validationException = ValidateActions.addValidationError("type is missing", validationException); } if (id == null) { - validationException = Actions.addValidationError("id is missing", validationException); + validationException = ValidateActions.addValidationError("id is missing", validationException); } return validationException; } diff --git a/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java b/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java index 10bba6e0dec..361c167d557 100644 --- a/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java +++ b/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java @@ -22,7 +22,7 @@ package org.elasticsearch.action.get; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.Actions; +import org.elasticsearch.action.ValidateActions; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -186,15 +186,15 @@ public class MultiGetRequest implements ActionRequest { public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (items.isEmpty()) { - validationException = Actions.addValidationError("no documents to get", validationException); + validationException = ValidateActions.addValidationError("no documents to get", validationException); } else { for (int i = 0; i < items.size(); i++) { Item item = items.get(i); if (item.index() == null) { - validationException = Actions.addValidationError("index is missing for doc " + i, validationException); + validationException = ValidateActions.addValidationError("index is missing for doc " + i, validationException); } if (item.id() == null) { - validationException = Actions.addValidationError("id is missing for doc " + i, validationException); + validationException = ValidateActions.addValidationError("id is missing for doc " + i, validationException); } } } diff --git a/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/src/main/java/org/elasticsearch/action/index/IndexRequest.java index a674217b75c..7c14c980427 100644 --- a/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -47,7 +47,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Map; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; /** * Index request to index a typed JSON document into a specific index and make it searchable. Best diff --git a/src/main/java/org/elasticsearch/action/mlt/MoreLikeThisRequest.java b/src/main/java/org/elasticsearch/action/mlt/MoreLikeThisRequest.java index 7d26810ddaa..77e4555d16b 100644 --- a/src/main/java/org/elasticsearch/action/mlt/MoreLikeThisRequest.java +++ b/src/main/java/org/elasticsearch/action/mlt/MoreLikeThisRequest.java @@ -24,7 +24,7 @@ import org.elasticsearch.ElasticSearchGenerationException; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.Actions; +import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Requests; import org.elasticsearch.common.Bytes; @@ -52,7 +52,6 @@ import static org.elasticsearch.search.Scroll.readScroll; *

    *

    Note, the {@link #index()}, {@link #type(String)} and {@link #id(String)} are required. * - * * @see org.elasticsearch.client.Client#moreLikeThis(MoreLikeThisRequest) * @see org.elasticsearch.client.Requests#moreLikeThisRequest(String) * @see org.elasticsearch.action.search.SearchResponse @@ -526,13 +525,13 @@ public class MoreLikeThisRequest implements ActionRequest { public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (index == null) { - validationException = Actions.addValidationError("index is missing", validationException); + validationException = ValidateActions.addValidationError("index is missing", validationException); } if (type == null) { - validationException = Actions.addValidationError("type is missing", validationException); + validationException = ValidateActions.addValidationError("type is missing", validationException); } if (id == null) { - validationException = Actions.addValidationError("id is missing", validationException); + validationException = ValidateActions.addValidationError("id is missing", validationException); } return validationException; } diff --git a/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java b/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java index cb4916670f8..b83be4bdc09 100644 --- a/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java +++ b/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java @@ -36,7 +36,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Map; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; /** * diff --git a/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 2727b8df6bc..c441da51a92 100644 --- a/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -43,7 +43,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Map; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.search.Scroll.readScroll; /** diff --git a/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java b/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java index 55ebce5c7d6..c70b3f417bc 100644 --- a/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java +++ b/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java @@ -28,7 +28,7 @@ import org.elasticsearch.search.Scroll; import java.io.IOException; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.search.Scroll.readScroll; /** diff --git a/src/main/java/org/elasticsearch/action/support/replication/IndexReplicationOperationRequest.java b/src/main/java/org/elasticsearch/action/support/replication/IndexReplicationOperationRequest.java index ef57723f327..130ed65cf66 100644 --- a/src/main/java/org/elasticsearch/action/support/replication/IndexReplicationOperationRequest.java +++ b/src/main/java/org/elasticsearch/action/support/replication/IndexReplicationOperationRequest.java @@ -28,7 +28,7 @@ import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; /** * diff --git a/src/main/java/org/elasticsearch/action/support/replication/ShardReplicationOperationRequest.java b/src/main/java/org/elasticsearch/action/support/replication/ShardReplicationOperationRequest.java index 722bafc55fc..6d01a91a2bf 100644 --- a/src/main/java/org/elasticsearch/action/support/replication/ShardReplicationOperationRequest.java +++ b/src/main/java/org/elasticsearch/action/support/replication/ShardReplicationOperationRequest.java @@ -29,7 +29,7 @@ import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; /** * diff --git a/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java b/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java index 5e1a3941eb8..410ac56c807 100644 --- a/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java +++ b/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java @@ -21,7 +21,7 @@ package org.elasticsearch.action.support.single.instance; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.Actions; +import org.elasticsearch.action.ValidateActions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; @@ -59,7 +59,7 @@ public abstract class InstanceShardOperationRequest implements ActionRequest { public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (index == null) { - validationException = Actions.addValidationError("index is missing", validationException); + validationException = ValidateActions.addValidationError("index is missing", validationException); } return validationException; } diff --git a/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequest.java b/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequest.java index 180b943f726..0d6aaa5daff 100644 --- a/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequest.java +++ b/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequest.java @@ -21,7 +21,7 @@ package org.elasticsearch.action.support.single.shard; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.Actions; +import org.elasticsearch.action.ValidateActions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -48,7 +48,7 @@ public abstract class SingleShardOperationRequest implements ActionRequest { public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (index == null) { - validationException = Actions.addValidationError("index is missing", validationException); + validationException = ValidateActions.addValidationError("index is missing", validationException); } return validationException; } diff --git a/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 9934c6f43b5..2dfe33cb435 100644 --- a/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -32,7 +32,7 @@ import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; import java.util.Map; -import static org.elasticsearch.action.Actions.addValidationError; +import static org.elasticsearch.action.ValidateActions.addValidationError; /** */ From c3c5aec567a89f7ce51919b80eee5e453d875d9a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 15 Jan 2012 12:14:57 +0200 Subject: [PATCH 168/270] rename RequestBuilder to ActionRequestBuilder and have a request() method on the actual interface --- .../{RequestBuilder.java => ActionRequestBuilder.java} | 4 +++- .../admin/cluster/support/BaseClusterRequestBuilder.java | 9 +++++++-- .../admin/indices/support/BaseIndicesRequestBuilder.java | 9 +++++++-- .../client/action/support/BaseRequestBuilder.java | 4 ++-- 4 files changed, 19 insertions(+), 7 deletions(-) rename src/main/java/org/elasticsearch/client/action/{RequestBuilder.java => ActionRequestBuilder.java} (90%) diff --git a/src/main/java/org/elasticsearch/client/action/RequestBuilder.java b/src/main/java/org/elasticsearch/client/action/ActionRequestBuilder.java similarity index 90% rename from src/main/java/org/elasticsearch/client/action/RequestBuilder.java rename to src/main/java/org/elasticsearch/client/action/ActionRequestBuilder.java index 5fc6baa867d..55de41a6c5c 100644 --- a/src/main/java/org/elasticsearch/client/action/RequestBuilder.java +++ b/src/main/java/org/elasticsearch/client/action/ActionRequestBuilder.java @@ -27,7 +27,9 @@ import org.elasticsearch.action.ListenableActionFuture; /** * */ -public interface RequestBuilder { +public interface ActionRequestBuilder { + + Request request(); ListenableActionFuture execute(); diff --git a/src/main/java/org/elasticsearch/client/action/admin/cluster/support/BaseClusterRequestBuilder.java b/src/main/java/org/elasticsearch/client/action/admin/cluster/support/BaseClusterRequestBuilder.java index 44c7a3954f0..c73ea90416c 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/cluster/support/BaseClusterRequestBuilder.java +++ b/src/main/java/org/elasticsearch/client/action/admin/cluster/support/BaseClusterRequestBuilder.java @@ -25,13 +25,13 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ListenableActionFuture; import org.elasticsearch.action.support.PlainListenableActionFuture; import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.client.action.RequestBuilder; +import org.elasticsearch.client.action.ActionRequestBuilder; import org.elasticsearch.client.internal.InternalClusterAdminClient; /** * */ -public abstract class BaseClusterRequestBuilder implements RequestBuilder { +public abstract class BaseClusterRequestBuilder implements ActionRequestBuilder { protected final InternalClusterAdminClient client; @@ -42,6 +42,11 @@ public abstract class BaseClusterRequestBuilder execute() { PlainListenableActionFuture future = new PlainListenableActionFuture(request.listenerThreaded(), client.threadPool()); diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/support/BaseIndicesRequestBuilder.java b/src/main/java/org/elasticsearch/client/action/admin/indices/support/BaseIndicesRequestBuilder.java index d5c47bfb0cd..b15d1e17028 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/support/BaseIndicesRequestBuilder.java +++ b/src/main/java/org/elasticsearch/client/action/admin/indices/support/BaseIndicesRequestBuilder.java @@ -25,13 +25,13 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ListenableActionFuture; import org.elasticsearch.action.support.PlainListenableActionFuture; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.RequestBuilder; +import org.elasticsearch.client.action.ActionRequestBuilder; import org.elasticsearch.client.internal.InternalIndicesAdminClient; /** * */ -public abstract class BaseIndicesRequestBuilder implements RequestBuilder { +public abstract class BaseIndicesRequestBuilder implements ActionRequestBuilder { protected final InternalIndicesAdminClient client; @@ -42,6 +42,11 @@ public abstract class BaseIndicesRequestBuilder execute() { PlainListenableActionFuture future = new PlainListenableActionFuture(request.listenerThreaded(), client.threadPool()); diff --git a/src/main/java/org/elasticsearch/client/action/support/BaseRequestBuilder.java b/src/main/java/org/elasticsearch/client/action/support/BaseRequestBuilder.java index 2066d9ea0cb..c16da2f177e 100644 --- a/src/main/java/org/elasticsearch/client/action/support/BaseRequestBuilder.java +++ b/src/main/java/org/elasticsearch/client/action/support/BaseRequestBuilder.java @@ -25,13 +25,13 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ListenableActionFuture; import org.elasticsearch.action.support.PlainListenableActionFuture; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.RequestBuilder; +import org.elasticsearch.client.action.ActionRequestBuilder; import org.elasticsearch.client.internal.InternalClient; /** * */ -public abstract class BaseRequestBuilder implements RequestBuilder { +public abstract class BaseRequestBuilder implements ActionRequestBuilder { protected final InternalClient client; From 8ee6ee05cd572307a4c3bbd13ea6c13993340ffb Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 15 Jan 2012 12:44:48 +0200 Subject: [PATCH 169/270] Java API: Move all request builders to org.elasticsearch.action... from org.elasticsearch.client.action, closes #1611. --- .../action/ActionRequestBuilder.java | 7 +--- .../health/ClusterHealthRequestBuilder.java | 7 +--- .../node/info/NodesInfoRequestBuilder.java | 6 +-- .../restart/NodesRestartRequestBuilder.java | 6 +-- .../shutdown/NodesShutdownRequestBuilder.java | 6 +-- .../node/stats/NodesStatsRequestBuilder.java | 6 +-- .../BroadcastPingRequestBuilder.java | 6 +-- .../ReplicationPingRequestBuilder.java | 6 +-- .../ping/single/SinglePingRequestBuilder.java | 6 +-- .../reroute/ClusterRerouteRequestBuilder.java | 6 +-- .../ClusterUpdateSettingsRequestBuilder.java | 6 +-- .../state/ClusterStateRequestBuilder.java | 6 +-- .../support/BaseClusterRequestBuilder.java | 8 +--- .../alias/IndicesAliasesRequestBuilder.java | 6 +-- .../analyze/AnalyzeRequestBuilder.java | 6 +-- .../ClearIndicesCacheRequestBuilder.java | 6 +-- .../close/CloseIndexRequestBuilder.java | 6 +-- .../create/CreateIndexRequestBuilder.java | 6 +-- .../delete/DeleteIndexRequestBuilder.java | 6 +-- .../exists/IndicesExistsRequestBuilder.java | 6 +-- .../indices/flush/FlushRequestBuilder.java | 6 +-- .../GatewaySnapshotRequestBuilder.java | 6 +-- .../delete/DeleteMappingRequestBuilder.java | 6 +-- .../mapping/put/PutMappingRequestBuilder.java | 6 +-- .../indices/open/OpenIndexRequestBuilder.java | 6 +-- .../optimize/OptimizeRequestBuilder.java | 8 +--- .../refresh/RefreshRequestBuilder.java | 8 +--- .../IndicesSegmentsRequestBuilder.java | 6 +-- .../UpdateSettingsRequestBuilder.java | 6 +-- .../stats/IndicesStatsRequestBuilder.java | 6 +-- .../status/IndicesStatusRequestBuilder.java | 6 +-- .../support/BaseIndicesRequestBuilder.java | 8 +--- .../DeleteIndexTemplateRequestBuilder.java | 6 +-- .../put/PutIndexTemplateRequestBuilder.java | 6 +-- .../query/ValidateQueryRequestBuilder.java | 6 +-- .../action/bulk/BulkRequestBuilder.java | 12 ++---- .../action/count/CountRequestBuilder.java | 8 +--- .../action/delete/DeleteRequestBuilder.java | 8 +--- .../DeleteByQueryRequestBuilder.java | 6 +-- .../action/get/GetRequestBuilder.java | 8 +--- .../action/get/MultiGetRequestBuilder.java | 8 +--- .../action/index/IndexRequestBuilder.java | 8 +--- .../mlt/MoreLikeThisRequestBuilder.java | 5 +-- .../percolate/PercolateRequestBuilder.java | 6 +-- .../action/search/SearchRequestBuilder.java | 8 +--- .../search/SearchScrollRequestBuilder.java | 9 +--- .../action/support/BaseRequestBuilder.java | 9 +--- .../action/update/UpdateRequestBuilder.java | 6 +-- .../java/org/elasticsearch/client/Client.java | 29 +++++-------- .../client/ClusterAdminClient.java | 23 +++++----- .../client/IndicesAdminClient.java | 42 +++++++++---------- .../client/support/AbstractClient.java | 24 +++++------ .../support/AbstractClusterAdminClient.java | 22 +++++----- .../support/AbstractIndicesAdminClient.java | 42 +++++++++---------- .../indices/ttl/IndicesTTLService.java | 2 +- .../search/child/ChildSearchBenchmark.java | 2 +- .../facet/HistogramFacetSearchBenchmark.java | 2 +- .../QueryFilterFacetSearchBenchmark.java | 2 +- .../facet/TermsFacetSearchBenchmark.java | 2 +- .../stress/SingleThreadBulkStress.java | 2 +- .../indices/analyze/AnalyzeActionTests.java | 2 +- .../percolator/SimplePercolatorTests.java | 2 +- .../ClusterAndIndexReaderOnlyTests.java | 6 +-- .../fullrestart/FullRestartStressTest.java | 2 +- .../indexing/BulkIndexingStressTest.java | 2 +- .../stress/search1/ParentChildStressTest.java | 2 +- .../stress/search1/Search1StressTest.java | 2 +- 67 files changed, 198 insertions(+), 335 deletions(-) rename src/main/java/org/elasticsearch/{client => }/action/ActionRequestBuilder.java (81%) rename src/main/java/org/elasticsearch/{client => }/action/admin/cluster/health/ClusterHealthRequestBuilder.java (89%) rename src/main/java/org/elasticsearch/{client => }/action/admin/cluster/node/info/NodesInfoRequestBuilder.java (91%) rename src/main/java/org/elasticsearch/{client => }/action/admin/cluster/node/restart/NodesRestartRequestBuilder.java (86%) rename src/main/java/org/elasticsearch/{client => }/action/admin/cluster/node/shutdown/NodesShutdownRequestBuilder.java (88%) rename src/main/java/org/elasticsearch/{client => }/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java (90%) rename src/main/java/org/elasticsearch/{client => }/action/admin/cluster/ping/broadcast/BroadcastPingRequestBuilder.java (86%) rename src/main/java/org/elasticsearch/{client => }/action/admin/cluster/ping/replication/ReplicationPingRequestBuilder.java (86%) rename src/main/java/org/elasticsearch/{client => }/action/admin/cluster/ping/single/SinglePingRequestBuilder.java (86%) rename src/main/java/org/elasticsearch/{client => }/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java (86%) rename src/main/java/org/elasticsearch/{client => }/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java (91%) rename src/main/java/org/elasticsearch/{client => }/action/admin/cluster/state/ClusterStateRequestBuilder.java (91%) rename src/main/java/org/elasticsearch/{client => }/action/admin/cluster/support/BaseClusterRequestBuilder.java (86%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/alias/IndicesAliasesRequestBuilder.java (92%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/analyze/AnalyzeRequestBuilder.java (90%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java (89%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/close/CloseIndexRequestBuilder.java (89%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/create/CreateIndexRequestBuilder.java (94%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/delete/DeleteIndexRequestBuilder.java (89%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/exists/IndicesExistsRequestBuilder.java (83%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/flush/FlushRequestBuilder.java (85%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/gateway/snapshot/GatewaySnapshotRequestBuilder.java (81%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/mapping/delete/DeleteMappingRequestBuilder.java (85%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/mapping/put/PutMappingRequestBuilder.java (92%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/open/OpenIndexRequestBuilder.java (89%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/optimize/OptimizeRequestBuilder.java (92%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/refresh/RefreshRequestBuilder.java (89%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/segments/IndicesSegmentsRequestBuilder.java (82%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/settings/UpdateSettingsRequestBuilder.java (88%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/stats/IndicesStatsRequestBuilder.java (92%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/status/IndicesStatusRequestBuilder.java (86%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/support/BaseIndicesRequestBuilder.java (86%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java (88%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java (94%) rename src/main/java/org/elasticsearch/{client => }/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java (86%) rename src/main/java/org/elasticsearch/{client => }/action/bulk/BulkRequestBuilder.java (92%) rename src/main/java/org/elasticsearch/{client => }/action/count/CountRequestBuilder.java (94%) rename src/main/java/org/elasticsearch/{client => }/action/delete/DeleteRequestBuilder.java (94%) rename src/main/java/org/elasticsearch/{client => }/action/deletebyquery/DeleteByQueryRequestBuilder.java (95%) rename src/main/java/org/elasticsearch/{client => }/action/get/GetRequestBuilder.java (94%) rename src/main/java/org/elasticsearch/{client => }/action/get/MultiGetRequestBuilder.java (93%) rename src/main/java/org/elasticsearch/{client => }/action/index/IndexRequestBuilder.java (97%) rename src/main/java/org/elasticsearch/{client => }/action/mlt/MoreLikeThisRequestBuilder.java (97%) rename src/main/java/org/elasticsearch/{client => }/action/percolate/PercolateRequestBuilder.java (95%) rename src/main/java/org/elasticsearch/{client => }/action/search/SearchRequestBuilder.java (98%) rename src/main/java/org/elasticsearch/{client => }/action/search/SearchScrollRequestBuilder.java (89%) rename src/main/java/org/elasticsearch/{client => }/action/support/BaseRequestBuilder.java (82%) rename src/main/java/org/elasticsearch/{client => }/action/update/UpdateRequestBuilder.java (95%) diff --git a/src/main/java/org/elasticsearch/client/action/ActionRequestBuilder.java b/src/main/java/org/elasticsearch/action/ActionRequestBuilder.java similarity index 81% rename from src/main/java/org/elasticsearch/client/action/ActionRequestBuilder.java rename to src/main/java/org/elasticsearch/action/ActionRequestBuilder.java index 55de41a6c5c..a40663ef910 100644 --- a/src/main/java/org/elasticsearch/client/action/ActionRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/ActionRequestBuilder.java @@ -17,12 +17,7 @@ * under the License. */ -package org.elasticsearch.client.action; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.ListenableActionFuture; +package org.elasticsearch.action; /** * diff --git a/src/main/java/org/elasticsearch/client/action/admin/cluster/health/ClusterHealthRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java similarity index 89% rename from src/main/java/org/elasticsearch/client/action/admin/cluster/health/ClusterHealthRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java index 00dd52a3df5..df62b48a564 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/cluster/health/ClusterHealthRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java @@ -17,14 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.cluster.health; +package org.elasticsearch.action.admin.cluster.health; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.client.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.common.unit.TimeValue; /** diff --git a/src/main/java/org/elasticsearch/client/action/admin/cluster/node/info/NodesInfoRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java similarity index 91% rename from src/main/java/org/elasticsearch/client/action/admin/cluster/node/info/NodesInfoRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java index aad47e0ee4d..87723b48020 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/cluster/node/info/NodesInfoRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.cluster.node.info; +package org.elasticsearch.action.admin.cluster.node.info; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.client.action.admin.cluster.support.BaseClusterRequestBuilder; /** * diff --git a/src/main/java/org/elasticsearch/client/action/admin/cluster/node/restart/NodesRestartRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/restart/NodesRestartRequestBuilder.java similarity index 86% rename from src/main/java/org/elasticsearch/client/action/admin/cluster/node/restart/NodesRestartRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/cluster/node/restart/NodesRestartRequestBuilder.java index d8ddb9a71b4..bf06fbaf8bc 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/cluster/node/restart/NodesRestartRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/restart/NodesRestartRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.cluster.node.restart; +package org.elasticsearch.action.admin.cluster.node.restart; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartRequest; -import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartResponse; +import org.elasticsearch.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.client.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.common.unit.TimeValue; /** diff --git a/src/main/java/org/elasticsearch/client/action/admin/cluster/node/shutdown/NodesShutdownRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesShutdownRequestBuilder.java similarity index 88% rename from src/main/java/org/elasticsearch/client/action/admin/cluster/node/shutdown/NodesShutdownRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesShutdownRequestBuilder.java index 2c2ba568869..9a1b7514022 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/cluster/node/shutdown/NodesShutdownRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesShutdownRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.cluster.node.shutdown; +package org.elasticsearch.action.admin.cluster.node.shutdown; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownRequest; -import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownResponse; +import org.elasticsearch.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.client.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.common.unit.TimeValue; /** diff --git a/src/main/java/org/elasticsearch/client/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java similarity index 90% rename from src/main/java/org/elasticsearch/client/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java index 074d0647a8f..d5724bc7947 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.cluster.node.stats; +package org.elasticsearch.action.admin.cluster.node.stats; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.elasticsearch.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.client.action.admin.cluster.support.BaseClusterRequestBuilder; /** * diff --git a/src/main/java/org/elasticsearch/client/action/admin/cluster/ping/broadcast/BroadcastPingRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/BroadcastPingRequestBuilder.java similarity index 86% rename from src/main/java/org/elasticsearch/client/action/admin/cluster/ping/broadcast/BroadcastPingRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/BroadcastPingRequestBuilder.java index e0e63be6a05..e04703c49e7 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/cluster/ping/broadcast/BroadcastPingRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/BroadcastPingRequestBuilder.java @@ -17,14 +17,12 @@ * under the License. */ -package org.elasticsearch.client.action.admin.cluster.ping.broadcast; +package org.elasticsearch.action.admin.cluster.ping.broadcast; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingRequest; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingResponse; +import org.elasticsearch.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.client.action.admin.cluster.support.BaseClusterRequestBuilder; /** * diff --git a/src/main/java/org/elasticsearch/client/action/admin/cluster/ping/replication/ReplicationPingRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ReplicationPingRequestBuilder.java similarity index 86% rename from src/main/java/org/elasticsearch/client/action/admin/cluster/ping/replication/ReplicationPingRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ReplicationPingRequestBuilder.java index 624b62993ac..c3151391caa 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/cluster/ping/replication/ReplicationPingRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ReplicationPingRequestBuilder.java @@ -17,14 +17,12 @@ * under the License. */ -package org.elasticsearch.client.action.admin.cluster.ping.replication; +package org.elasticsearch.action.admin.cluster.ping.replication; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingRequest; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingResponse; +import org.elasticsearch.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.action.support.replication.ReplicationType; import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.client.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.common.unit.TimeValue; /** diff --git a/src/main/java/org/elasticsearch/client/action/admin/cluster/ping/single/SinglePingRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/SinglePingRequestBuilder.java similarity index 86% rename from src/main/java/org/elasticsearch/client/action/admin/cluster/ping/single/SinglePingRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/cluster/ping/single/SinglePingRequestBuilder.java index da73bffbc5b..ffd70b03e3c 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/cluster/ping/single/SinglePingRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/SinglePingRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.cluster.ping.single; +package org.elasticsearch.action.admin.cluster.ping.single; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingRequest; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingResponse; +import org.elasticsearch.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.client.action.admin.cluster.support.BaseClusterRequestBuilder; /** * diff --git a/src/main/java/org/elasticsearch/client/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java similarity index 86% rename from src/main/java/org/elasticsearch/client/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java index 50b0702b879..0748bedb1e8 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.cluster.reroute; +package org.elasticsearch.action.admin.cluster.reroute; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest; -import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; +import org.elasticsearch.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.client.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.common.unit.TimeValue; /** diff --git a/src/main/java/org/elasticsearch/client/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java similarity index 91% rename from src/main/java/org/elasticsearch/client/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java index d34b59c7334..c343d836460 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.cluster.settings; +package org.elasticsearch.action.admin.cluster.settings; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; -import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.elasticsearch.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.client.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; diff --git a/src/main/java/org/elasticsearch/client/action/admin/cluster/state/ClusterStateRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java similarity index 91% rename from src/main/java/org/elasticsearch/client/action/admin/cluster/state/ClusterStateRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java index c8cb3c0ffd8..bf5f997bd8b 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/cluster/state/ClusterStateRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.cluster.state; +package org.elasticsearch.action.admin.cluster.state; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.client.action.admin.cluster.support.BaseClusterRequestBuilder; import org.elasticsearch.common.unit.TimeValue; /** diff --git a/src/main/java/org/elasticsearch/client/action/admin/cluster/support/BaseClusterRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/support/BaseClusterRequestBuilder.java similarity index 86% rename from src/main/java/org/elasticsearch/client/action/admin/cluster/support/BaseClusterRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/cluster/support/BaseClusterRequestBuilder.java index c73ea90416c..1318a7cb398 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/cluster/support/BaseClusterRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/support/BaseClusterRequestBuilder.java @@ -17,15 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.cluster.support; +package org.elasticsearch.action.admin.cluster.support; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.*; import org.elasticsearch.action.support.PlainListenableActionFuture; import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.client.action.ActionRequestBuilder; import org.elasticsearch.client.internal.InternalClusterAdminClient; /** diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/alias/IndicesAliasesRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java similarity index 92% rename from src/main/java/org/elasticsearch/client/action/admin/indices/alias/IndicesAliasesRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java index 96d82314681..99e7ceefca9 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/alias/IndicesAliasesRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.alias; +package org.elasticsearch.action.admin.indices.alias; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.cluster.metadata.AliasAction; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.FilterBuilder; diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/analyze/AnalyzeRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java similarity index 90% rename from src/main/java/org/elasticsearch/client/action/admin/indices/analyze/AnalyzeRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java index 0753879ae4a..8fa274268f9 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/analyze/AnalyzeRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.analyze; +package org.elasticsearch.action.admin.indices.analyze; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; /** * diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java similarity index 89% rename from src/main/java/org/elasticsearch/client/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java index 448ed2b38a6..5fa2eaa74b9 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java @@ -17,14 +17,12 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.cache.clear; +package org.elasticsearch.action.admin.indices.cache.clear; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; /** * diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/close/CloseIndexRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestBuilder.java similarity index 89% rename from src/main/java/org/elasticsearch/client/action/admin/indices/close/CloseIndexRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestBuilder.java index 15a7863442a..656c656b38e 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/close/CloseIndexRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.close; +package org.elasticsearch.action.admin.indices.close; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; -import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.common.unit.TimeValue; /** diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/create/CreateIndexRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java similarity index 94% rename from src/main/java/org/elasticsearch/client/action/admin/indices/create/CreateIndexRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java index 75a14c30c4b..0af6832f6e7 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/create/CreateIndexRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.create; +package org.elasticsearch.action.admin.indices.create; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/delete/DeleteIndexRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java similarity index 89% rename from src/main/java/org/elasticsearch/client/action/admin/indices/delete/DeleteIndexRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java index 432132e4633..510eed6f779 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/delete/DeleteIndexRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.delete; +package org.elasticsearch.action.admin.indices.delete; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.common.unit.TimeValue; /** diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/exists/IndicesExistsRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsRequestBuilder.java similarity index 83% rename from src/main/java/org/elasticsearch/client/action/admin/indices/exists/IndicesExistsRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsRequestBuilder.java index 494c36e2228..5385a7b4b30 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/exists/IndicesExistsRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.exists; +package org.elasticsearch.action.admin.indices.exists; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.exists.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.IndicesExistsResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; /** * diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/flush/FlushRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequestBuilder.java similarity index 85% rename from src/main/java/org/elasticsearch/client/action/admin/indices/flush/FlushRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequestBuilder.java index 5de79176965..db773010eed 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/flush/FlushRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.flush; +package org.elasticsearch.action.admin.indices.flush; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.flush.FlushRequest; -import org.elasticsearch.action.admin.indices.flush.FlushResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; /** * diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/gateway/snapshot/GatewaySnapshotRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/gateway/snapshot/GatewaySnapshotRequestBuilder.java similarity index 81% rename from src/main/java/org/elasticsearch/client/action/admin/indices/gateway/snapshot/GatewaySnapshotRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/gateway/snapshot/GatewaySnapshotRequestBuilder.java index 0302333ea8c..8e5759433e9 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/gateway/snapshot/GatewaySnapshotRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/gateway/snapshot/GatewaySnapshotRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.gateway.snapshot; +package org.elasticsearch.action.admin.indices.gateway.snapshot; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotRequest; -import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; /** * diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/mapping/delete/DeleteMappingRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/DeleteMappingRequestBuilder.java similarity index 85% rename from src/main/java/org/elasticsearch/client/action/admin/indices/mapping/delete/DeleteMappingRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/DeleteMappingRequestBuilder.java index 05bc9818be1..c3f0739da1d 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/mapping/delete/DeleteMappingRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/DeleteMappingRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.mapping.delete; +package org.elasticsearch.action.admin.indices.mapping.delete; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.common.unit.TimeValue; /** diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/mapping/put/PutMappingRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java similarity index 92% rename from src/main/java/org/elasticsearch/client/action/admin/indices/mapping/put/PutMappingRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java index c4cac43cb8f..76301b5a9a2 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/mapping/put/PutMappingRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.mapping.put; +package org.elasticsearch.action.admin.indices.mapping.put; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.common.Required; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/open/OpenIndexRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequestBuilder.java similarity index 89% rename from src/main/java/org/elasticsearch/client/action/admin/indices/open/OpenIndexRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequestBuilder.java index 0a4299b7b6f..64f425dfeb8 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/open/OpenIndexRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.open; +package org.elasticsearch.action.admin.indices.open; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; -import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.common.unit.TimeValue; /** diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/optimize/OptimizeRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeRequestBuilder.java similarity index 92% rename from src/main/java/org/elasticsearch/client/action/admin/indices/optimize/OptimizeRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeRequestBuilder.java index 5ded31f388d..ed8b4d3ba18 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/optimize/OptimizeRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeRequestBuilder.java @@ -17,14 +17,12 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.optimize; +package org.elasticsearch.action.admin.indices.optimize; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest; -import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; /** * A request to optimize one or more indices. In order to optimize on all the indices, pass an empty array or @@ -35,8 +33,6 @@ import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestB *

    *

    {@link #setMaxNumSegments(int)} allows to control the number of segments to optimize down to. By default, will * cause the optimize process to optimize down to half the configured number of segments. - * - * */ public class OptimizeRequestBuilder extends BaseIndicesRequestBuilder { diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/refresh/RefreshRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshRequestBuilder.java similarity index 89% rename from src/main/java/org/elasticsearch/client/action/admin/indices/refresh/RefreshRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshRequestBuilder.java index b0e33065bb0..228964cd85d 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/refresh/RefreshRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshRequestBuilder.java @@ -17,21 +17,17 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.refresh; +package org.elasticsearch.action.admin.indices.refresh; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; /** * A refresh request making all operations performed since the last refresh available for search. The (near) real-time * capabilities depends on the index engine used. For example, the robin one requires refresh to be called, but by * default a refresh is scheduled periodically. - * - * */ public class RefreshRequestBuilder extends BaseIndicesRequestBuilder { diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/segments/IndicesSegmentsRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestBuilder.java similarity index 82% rename from src/main/java/org/elasticsearch/client/action/admin/indices/segments/IndicesSegmentsRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestBuilder.java index 8c8214e30eb..6a511f2e725 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/segments/IndicesSegmentsRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.segments; +package org.elasticsearch.action.admin.indices.segments; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; -import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; /** * diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/settings/UpdateSettingsRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/settings/UpdateSettingsRequestBuilder.java similarity index 88% rename from src/main/java/org/elasticsearch/client/action/admin/indices/settings/UpdateSettingsRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/settings/UpdateSettingsRequestBuilder.java index c420cfac3b4..5d86f35b33d 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/settings/UpdateSettingsRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/settings/UpdateSettingsRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.settings; +package org.elasticsearch.action.admin.indices.settings; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.settings.UpdateSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.UpdateSettingsResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.common.settings.Settings; import java.util.Map; diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/stats/IndicesStatsRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java similarity index 92% rename from src/main/java/org/elasticsearch/client/action/admin/indices/stats/IndicesStatsRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java index 5a894b4fc28..b44a78faa0b 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/stats/IndicesStatsRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.stats; +package org.elasticsearch.action.admin.indices.stats; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.stats.IndicesStats; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; /** * A request to get indices level stats. Allow to enable different stats to be returned. diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/status/IndicesStatusRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/status/IndicesStatusRequestBuilder.java similarity index 86% rename from src/main/java/org/elasticsearch/client/action/admin/indices/status/IndicesStatusRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/status/IndicesStatusRequestBuilder.java index 9b01f8fc75d..4540e4a7fd7 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/status/IndicesStatusRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/status/IndicesStatusRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.status; +package org.elasticsearch.action.admin.indices.status; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.status.IndicesStatusRequest; -import org.elasticsearch.action.admin.indices.status.IndicesStatusResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; /** * diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/support/BaseIndicesRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/support/BaseIndicesRequestBuilder.java similarity index 86% rename from src/main/java/org/elasticsearch/client/action/admin/indices/support/BaseIndicesRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/support/BaseIndicesRequestBuilder.java index b15d1e17028..30462a903e9 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/support/BaseIndicesRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/support/BaseIndicesRequestBuilder.java @@ -17,15 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.support; +package org.elasticsearch.action.admin.indices.support; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.*; import org.elasticsearch.action.support.PlainListenableActionFuture; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.ActionRequestBuilder; import org.elasticsearch.client.internal.InternalIndicesAdminClient; /** diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java similarity index 88% rename from src/main/java/org/elasticsearch/client/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java index 92a34a23599..d4c8c14d1a0 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.template.delete; +package org.elasticsearch.action.admin.indices.template.delete; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.common.unit.TimeValue; /** diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java similarity index 94% rename from src/main/java/org/elasticsearch/client/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java index 16c71e86687..6c1c7a737cd 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.admin.indices.template.put; +package org.elasticsearch.action.admin.indices.template.put; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; diff --git a/src/main/java/org/elasticsearch/client/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java similarity index 86% rename from src/main/java/org/elasticsearch/client/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java rename to src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java index 65c3399a2af..02982e685fe 100644 --- a/src/main/java/org/elasticsearch/client/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java @@ -1,11 +1,9 @@ -package org.elasticsearch.client.action.admin.indices.validate.query; +package org.elasticsearch.action.admin.indices.validate.query; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; -import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; +import org.elasticsearch.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.client.action.admin.indices.support.BaseIndicesRequestBuilder; import org.elasticsearch.index.query.QueryBuilder; /** diff --git a/src/main/java/org/elasticsearch/client/action/bulk/BulkRequestBuilder.java b/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java similarity index 92% rename from src/main/java/org/elasticsearch/client/action/bulk/BulkRequestBuilder.java rename to src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java index ac7ec8ef1b9..0013d039104 100644 --- a/src/main/java/org/elasticsearch/client/action/bulk/BulkRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java @@ -17,26 +17,22 @@ * under the License. */ -package org.elasticsearch.client.action.bulk; +package org.elasticsearch.action.bulk; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.WriteConsistencyLevel; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteRequestBuilder; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.support.BaseRequestBuilder; import org.elasticsearch.action.support.replication.ReplicationType; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.delete.DeleteRequestBuilder; -import org.elasticsearch.client.action.index.IndexRequestBuilder; -import org.elasticsearch.client.action.support.BaseRequestBuilder; import org.elasticsearch.common.Nullable; /** * A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes * it in a single batch. - * - * */ public class BulkRequestBuilder extends BaseRequestBuilder { diff --git a/src/main/java/org/elasticsearch/client/action/count/CountRequestBuilder.java b/src/main/java/org/elasticsearch/action/count/CountRequestBuilder.java similarity index 94% rename from src/main/java/org/elasticsearch/client/action/count/CountRequestBuilder.java rename to src/main/java/org/elasticsearch/action/count/CountRequestBuilder.java index d3d041a286b..34f79e7e38f 100644 --- a/src/main/java/org/elasticsearch/client/action/count/CountRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/count/CountRequestBuilder.java @@ -17,20 +17,16 @@ * under the License. */ -package org.elasticsearch.client.action.count; +package org.elasticsearch.action.count; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.count.CountRequest; -import org.elasticsearch.action.count.CountResponse; +import org.elasticsearch.action.support.BaseRequestBuilder; import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.support.BaseRequestBuilder; import org.elasticsearch.index.query.QueryBuilder; /** * A count action request builder. - * - * */ public class CountRequestBuilder extends BaseRequestBuilder { diff --git a/src/main/java/org/elasticsearch/client/action/delete/DeleteRequestBuilder.java b/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java similarity index 94% rename from src/main/java/org/elasticsearch/client/action/delete/DeleteRequestBuilder.java rename to src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java index 647267c2a00..c2bffc94d62 100644 --- a/src/main/java/org/elasticsearch/client/action/delete/DeleteRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java @@ -17,22 +17,18 @@ * under the License. */ -package org.elasticsearch.client.action.delete; +package org.elasticsearch.action.delete; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.WriteConsistencyLevel; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.support.BaseRequestBuilder; import org.elasticsearch.action.support.replication.ReplicationType; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.support.BaseRequestBuilder; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.VersionType; /** * A delete document action request builder. - * - * */ public class DeleteRequestBuilder extends BaseRequestBuilder { diff --git a/src/main/java/org/elasticsearch/client/action/deletebyquery/DeleteByQueryRequestBuilder.java b/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequestBuilder.java similarity index 95% rename from src/main/java/org/elasticsearch/client/action/deletebyquery/DeleteByQueryRequestBuilder.java rename to src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequestBuilder.java index 3cc32a8510b..60e19655455 100644 --- a/src/main/java/org/elasticsearch/client/action/deletebyquery/DeleteByQueryRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequestBuilder.java @@ -17,15 +17,13 @@ * under the License. */ -package org.elasticsearch.client.action.deletebyquery; +package org.elasticsearch.action.deletebyquery; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.WriteConsistencyLevel; -import org.elasticsearch.action.deletebyquery.DeleteByQueryRequest; -import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse; +import org.elasticsearch.action.support.BaseRequestBuilder; import org.elasticsearch.action.support.replication.ReplicationType; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.support.BaseRequestBuilder; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilder; diff --git a/src/main/java/org/elasticsearch/client/action/get/GetRequestBuilder.java b/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java similarity index 94% rename from src/main/java/org/elasticsearch/client/action/get/GetRequestBuilder.java rename to src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java index 983de0780bb..e61c3e344b0 100644 --- a/src/main/java/org/elasticsearch/client/action/get/GetRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java @@ -17,19 +17,15 @@ * under the License. */ -package org.elasticsearch.client.action.get; +package org.elasticsearch.action.get; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.support.BaseRequestBuilder; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.support.BaseRequestBuilder; import org.elasticsearch.common.Nullable; /** * A get document action request builder. - * - * */ public class GetRequestBuilder extends BaseRequestBuilder { diff --git a/src/main/java/org/elasticsearch/client/action/get/MultiGetRequestBuilder.java b/src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java similarity index 93% rename from src/main/java/org/elasticsearch/client/action/get/MultiGetRequestBuilder.java rename to src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java index 6e73f5e316c..39fe159d41a 100644 --- a/src/main/java/org/elasticsearch/client/action/get/MultiGetRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java @@ -17,19 +17,15 @@ * under the License. */ -package org.elasticsearch.client.action.get; +package org.elasticsearch.action.get; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.get.MultiGetRequest; -import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.support.BaseRequestBuilder; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.support.BaseRequestBuilder; import org.elasticsearch.common.Nullable; /** * A multi get document action request builder. - * - * */ public class MultiGetRequestBuilder extends BaseRequestBuilder { diff --git a/src/main/java/org/elasticsearch/client/action/index/IndexRequestBuilder.java b/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java similarity index 97% rename from src/main/java/org/elasticsearch/client/action/index/IndexRequestBuilder.java rename to src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index 98f134c024a..94837d4d5e2 100644 --- a/src/main/java/org/elasticsearch/client/action/index/IndexRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -17,15 +17,13 @@ * under the License. */ -package org.elasticsearch.client.action.index; +package org.elasticsearch.action.index; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.WriteConsistencyLevel; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.support.BaseRequestBuilder; import org.elasticsearch.action.support.replication.ReplicationType; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.support.BaseRequestBuilder; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -36,8 +34,6 @@ import java.util.Map; /** * An index document action request builder. - * - * */ public class IndexRequestBuilder extends BaseRequestBuilder { diff --git a/src/main/java/org/elasticsearch/client/action/mlt/MoreLikeThisRequestBuilder.java b/src/main/java/org/elasticsearch/action/mlt/MoreLikeThisRequestBuilder.java similarity index 97% rename from src/main/java/org/elasticsearch/client/action/mlt/MoreLikeThisRequestBuilder.java rename to src/main/java/org/elasticsearch/action/mlt/MoreLikeThisRequestBuilder.java index 5b24cc2c687..3e34539b3f5 100644 --- a/src/main/java/org/elasticsearch/client/action/mlt/MoreLikeThisRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/mlt/MoreLikeThisRequestBuilder.java @@ -17,15 +17,14 @@ * under the License. */ -package org.elasticsearch.client.action.mlt; +package org.elasticsearch.action.mlt; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.mlt.MoreLikeThisRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.BaseRequestBuilder; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.support.BaseRequestBuilder; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; diff --git a/src/main/java/org/elasticsearch/client/action/percolate/PercolateRequestBuilder.java b/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java similarity index 95% rename from src/main/java/org/elasticsearch/client/action/percolate/PercolateRequestBuilder.java rename to src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java index 9770ceecaf9..5d17c4b9437 100644 --- a/src/main/java/org/elasticsearch/client/action/percolate/PercolateRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java @@ -17,13 +17,11 @@ * under the License. */ -package org.elasticsearch.client.action.percolate; +package org.elasticsearch.action.percolate; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.percolate.PercolateRequest; -import org.elasticsearch.action.percolate.PercolateResponse; +import org.elasticsearch.action.support.BaseRequestBuilder; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.support.BaseRequestBuilder; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; diff --git a/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java b/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java similarity index 98% rename from src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java rename to src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java index 6428625162c..7a69053243a 100644 --- a/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java @@ -17,16 +17,12 @@ * under the License. */ -package org.elasticsearch.client.action.search; +package org.elasticsearch.action.search; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.search.SearchOperationThreading; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.BaseRequestBuilder; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.support.BaseRequestBuilder; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; diff --git a/src/main/java/org/elasticsearch/client/action/search/SearchScrollRequestBuilder.java b/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java similarity index 89% rename from src/main/java/org/elasticsearch/client/action/search/SearchScrollRequestBuilder.java rename to src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java index 743d3dc4892..4d92029ed56 100644 --- a/src/main/java/org/elasticsearch/client/action/search/SearchScrollRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java @@ -17,21 +17,16 @@ * under the License. */ -package org.elasticsearch.client.action.search; +package org.elasticsearch.action.search; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.search.SearchOperationThreading; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.action.support.BaseRequestBuilder; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.support.BaseRequestBuilder; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.Scroll; /** * A search scroll action request builder. - * - * */ public class SearchScrollRequestBuilder extends BaseRequestBuilder { diff --git a/src/main/java/org/elasticsearch/client/action/support/BaseRequestBuilder.java b/src/main/java/org/elasticsearch/action/support/BaseRequestBuilder.java similarity index 82% rename from src/main/java/org/elasticsearch/client/action/support/BaseRequestBuilder.java rename to src/main/java/org/elasticsearch/action/support/BaseRequestBuilder.java index c16da2f177e..f4a5d1c9a91 100644 --- a/src/main/java/org/elasticsearch/client/action/support/BaseRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/support/BaseRequestBuilder.java @@ -17,15 +17,10 @@ * under the License. */ -package org.elasticsearch.client.action.support; +package org.elasticsearch.action.support; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.support.PlainListenableActionFuture; +import org.elasticsearch.action.*; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.ActionRequestBuilder; import org.elasticsearch.client.internal.InternalClient; /** diff --git a/src/main/java/org/elasticsearch/client/action/update/UpdateRequestBuilder.java b/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java similarity index 95% rename from src/main/java/org/elasticsearch/client/action/update/UpdateRequestBuilder.java rename to src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java index dd5c4e3e90d..6b19d66ce83 100644 --- a/src/main/java/org/elasticsearch/client/action/update/UpdateRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java @@ -17,15 +17,13 @@ * under the License. */ -package org.elasticsearch.client.action.update; +package org.elasticsearch.action.update; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.WriteConsistencyLevel; +import org.elasticsearch.action.support.BaseRequestBuilder; import org.elasticsearch.action.support.replication.ReplicationType; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.support.BaseRequestBuilder; import org.elasticsearch.common.unit.TimeValue; import java.util.Map; diff --git a/src/main/java/org/elasticsearch/client/Client.java b/src/main/java/org/elasticsearch/client/Client.java index e72ef413c72..6e019eab6d1 100644 --- a/src/main/java/org/elasticsearch/client/Client.java +++ b/src/main/java/org/elasticsearch/client/Client.java @@ -22,39 +22,30 @@ package org.elasticsearch.client; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.count.CountRequest; +import org.elasticsearch.action.count.CountRequestBuilder; import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteRequestBuilder; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.deletebyquery.DeleteByQueryRequest; +import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder; import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse; -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.get.MultiGetRequest; -import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.get.*; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.mlt.MoreLikeThisRequest; +import org.elasticsearch.action.mlt.MoreLikeThisRequestBuilder; import org.elasticsearch.action.percolate.PercolateRequest; +import org.elasticsearch.action.percolate.PercolateRequestBuilder; import org.elasticsearch.action.percolate.PercolateResponse; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.action.search.*; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; -import org.elasticsearch.client.action.bulk.BulkRequestBuilder; -import org.elasticsearch.client.action.count.CountRequestBuilder; -import org.elasticsearch.client.action.delete.DeleteRequestBuilder; -import org.elasticsearch.client.action.deletebyquery.DeleteByQueryRequestBuilder; -import org.elasticsearch.client.action.get.GetRequestBuilder; -import org.elasticsearch.client.action.get.MultiGetRequestBuilder; -import org.elasticsearch.client.action.index.IndexRequestBuilder; -import org.elasticsearch.client.action.mlt.MoreLikeThisRequestBuilder; -import org.elasticsearch.client.action.percolate.PercolateRequestBuilder; -import org.elasticsearch.client.action.search.SearchRequestBuilder; -import org.elasticsearch.client.action.search.SearchScrollRequestBuilder; -import org.elasticsearch.client.action.update.UpdateRequestBuilder; import org.elasticsearch.common.Nullable; /** diff --git a/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/src/main/java/org/elasticsearch/client/ClusterAdminClient.java index a56e753bc6d..3784ba2f564 100644 --- a/src/main/java/org/elasticsearch/client/ClusterAdminClient.java +++ b/src/main/java/org/elasticsearch/client/ClusterAdminClient.java @@ -22,43 +22,42 @@ package org.elasticsearch.client; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequestBuilder; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartRequest; +import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartRequestBuilder; import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartResponse; import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownRequest; +import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownRequestBuilder; import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingRequest; +import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingRequestBuilder; import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingResponse; import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingRequest; +import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingRequestBuilder; import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingResponse; import org.elasticsearch.action.admin.cluster.ping.single.SinglePingRequest; +import org.elasticsearch.action.admin.cluster.ping.single.SinglePingRequestBuilder; import org.elasticsearch.action.admin.cluster.ping.single.SinglePingResponse; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest; +import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequestBuilder; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequestBuilder; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.client.action.admin.cluster.health.ClusterHealthRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.node.info.NodesInfoRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.node.restart.NodesRestartRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.node.shutdown.NodesShutdownRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.node.stats.NodesStatsRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.ping.broadcast.BroadcastPingRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.ping.replication.ReplicationPingRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.ping.single.SinglePingRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.reroute.ClusterRerouteRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.settings.ClusterUpdateSettingsRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.state.ClusterStateRequestBuilder; /** * Administrative actions/operations against indices. * - * * @see AdminClient#cluster() */ public interface ClusterAdminClient { diff --git a/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/src/main/java/org/elasticsearch/client/IndicesAdminClient.java index 323de8a8a8a..1ee20f70650 100644 --- a/src/main/java/org/elasticsearch/client/IndicesAdminClient.java +++ b/src/main/java/org/elasticsearch/client/IndicesAdminClient.java @@ -22,68 +22,68 @@ package org.elasticsearch.client; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder; import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; +import org.elasticsearch.action.admin.indices.close.CloseIndexRequestBuilder; import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.action.admin.indices.exists.IndicesExistsRequest; +import org.elasticsearch.action.admin.indices.exists.IndicesExistsRequestBuilder; import org.elasticsearch.action.admin.indices.exists.IndicesExistsResponse; import org.elasticsearch.action.admin.indices.flush.FlushRequest; +import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder; import org.elasticsearch.action.admin.indices.flush.FlushResponse; import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotRequest; +import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotRequestBuilder; import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotResponse; import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequest; +import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequestBuilder; import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; +import org.elasticsearch.action.admin.indices.open.OpenIndexRequestBuilder; import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest; +import org.elasticsearch.action.admin.indices.optimize.OptimizeRequestBuilder; import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest; +import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequestBuilder; import org.elasticsearch.action.admin.indices.settings.UpdateSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.UpdateSettingsRequestBuilder; import org.elasticsearch.action.admin.indices.settings.UpdateSettingsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStats; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; import org.elasticsearch.action.admin.indices.status.IndicesStatusRequest; +import org.elasticsearch.action.admin.indices.status.IndicesStatusRequestBuilder; import org.elasticsearch.action.admin.indices.status.IndicesStatusResponse; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; -import org.elasticsearch.client.action.admin.indices.alias.IndicesAliasesRequestBuilder; -import org.elasticsearch.client.action.admin.indices.analyze.AnalyzeRequestBuilder; -import org.elasticsearch.client.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder; -import org.elasticsearch.client.action.admin.indices.close.CloseIndexRequestBuilder; -import org.elasticsearch.client.action.admin.indices.create.CreateIndexRequestBuilder; -import org.elasticsearch.client.action.admin.indices.delete.DeleteIndexRequestBuilder; -import org.elasticsearch.client.action.admin.indices.exists.IndicesExistsRequestBuilder; -import org.elasticsearch.client.action.admin.indices.flush.FlushRequestBuilder; -import org.elasticsearch.client.action.admin.indices.gateway.snapshot.GatewaySnapshotRequestBuilder; -import org.elasticsearch.client.action.admin.indices.mapping.delete.DeleteMappingRequestBuilder; -import org.elasticsearch.client.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.client.action.admin.indices.open.OpenIndexRequestBuilder; -import org.elasticsearch.client.action.admin.indices.optimize.OptimizeRequestBuilder; -import org.elasticsearch.client.action.admin.indices.refresh.RefreshRequestBuilder; -import org.elasticsearch.client.action.admin.indices.segments.IndicesSegmentsRequestBuilder; -import org.elasticsearch.client.action.admin.indices.settings.UpdateSettingsRequestBuilder; -import org.elasticsearch.client.action.admin.indices.stats.IndicesStatsRequestBuilder; -import org.elasticsearch.client.action.admin.indices.status.IndicesStatusRequestBuilder; -import org.elasticsearch.client.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder; -import org.elasticsearch.client.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; -import org.elasticsearch.client.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.elasticsearch.common.Nullable; /** diff --git a/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/src/main/java/org/elasticsearch/client/support/AbstractClient.java index 4012ad515db..add13aa15b7 100644 --- a/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -19,18 +19,18 @@ package org.elasticsearch.client.support; -import org.elasticsearch.client.action.bulk.BulkRequestBuilder; -import org.elasticsearch.client.action.count.CountRequestBuilder; -import org.elasticsearch.client.action.delete.DeleteRequestBuilder; -import org.elasticsearch.client.action.deletebyquery.DeleteByQueryRequestBuilder; -import org.elasticsearch.client.action.get.GetRequestBuilder; -import org.elasticsearch.client.action.get.MultiGetRequestBuilder; -import org.elasticsearch.client.action.index.IndexRequestBuilder; -import org.elasticsearch.client.action.mlt.MoreLikeThisRequestBuilder; -import org.elasticsearch.client.action.percolate.PercolateRequestBuilder; -import org.elasticsearch.client.action.search.SearchRequestBuilder; -import org.elasticsearch.client.action.search.SearchScrollRequestBuilder; -import org.elasticsearch.client.action.update.UpdateRequestBuilder; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.count.CountRequestBuilder; +import org.elasticsearch.action.delete.DeleteRequestBuilder; +import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder; +import org.elasticsearch.action.get.GetRequestBuilder; +import org.elasticsearch.action.get.MultiGetRequestBuilder; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.mlt.MoreLikeThisRequestBuilder; +import org.elasticsearch.action.percolate.PercolateRequestBuilder; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.client.internal.InternalClient; import org.elasticsearch.common.Nullable; diff --git a/src/main/java/org/elasticsearch/client/support/AbstractClusterAdminClient.java b/src/main/java/org/elasticsearch/client/support/AbstractClusterAdminClient.java index 2475b63c6e9..2fce446801a 100644 --- a/src/main/java/org/elasticsearch/client/support/AbstractClusterAdminClient.java +++ b/src/main/java/org/elasticsearch/client/support/AbstractClusterAdminClient.java @@ -19,17 +19,17 @@ package org.elasticsearch.client.support; -import org.elasticsearch.client.action.admin.cluster.health.ClusterHealthRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.node.info.NodesInfoRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.node.restart.NodesRestartRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.node.shutdown.NodesShutdownRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.node.stats.NodesStatsRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.ping.broadcast.BroadcastPingRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.ping.replication.ReplicationPingRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.ping.single.SinglePingRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.reroute.ClusterRerouteRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.settings.ClusterUpdateSettingsRequestBuilder; -import org.elasticsearch.client.action.admin.cluster.state.ClusterStateRequestBuilder; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequestBuilder; +import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartRequestBuilder; +import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownRequestBuilder; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder; +import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingRequestBuilder; +import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingRequestBuilder; +import org.elasticsearch.action.admin.cluster.ping.single.SinglePingRequestBuilder; +import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequestBuilder; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequestBuilder; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; import org.elasticsearch.client.internal.InternalClusterAdminClient; /** diff --git a/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java b/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java index 04356b9be77..fb81126e924 100644 --- a/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java +++ b/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java @@ -19,27 +19,27 @@ package org.elasticsearch.client.support; -import org.elasticsearch.client.action.admin.indices.alias.IndicesAliasesRequestBuilder; -import org.elasticsearch.client.action.admin.indices.analyze.AnalyzeRequestBuilder; -import org.elasticsearch.client.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder; -import org.elasticsearch.client.action.admin.indices.close.CloseIndexRequestBuilder; -import org.elasticsearch.client.action.admin.indices.create.CreateIndexRequestBuilder; -import org.elasticsearch.client.action.admin.indices.delete.DeleteIndexRequestBuilder; -import org.elasticsearch.client.action.admin.indices.exists.IndicesExistsRequestBuilder; -import org.elasticsearch.client.action.admin.indices.flush.FlushRequestBuilder; -import org.elasticsearch.client.action.admin.indices.gateway.snapshot.GatewaySnapshotRequestBuilder; -import org.elasticsearch.client.action.admin.indices.mapping.delete.DeleteMappingRequestBuilder; -import org.elasticsearch.client.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.client.action.admin.indices.open.OpenIndexRequestBuilder; -import org.elasticsearch.client.action.admin.indices.optimize.OptimizeRequestBuilder; -import org.elasticsearch.client.action.admin.indices.refresh.RefreshRequestBuilder; -import org.elasticsearch.client.action.admin.indices.segments.IndicesSegmentsRequestBuilder; -import org.elasticsearch.client.action.admin.indices.settings.UpdateSettingsRequestBuilder; -import org.elasticsearch.client.action.admin.indices.stats.IndicesStatsRequestBuilder; -import org.elasticsearch.client.action.admin.indices.status.IndicesStatusRequestBuilder; -import org.elasticsearch.client.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder; -import org.elasticsearch.client.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; -import org.elasticsearch.client.action.admin.indices.validate.query.ValidateQueryRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder; +import org.elasticsearch.action.admin.indices.close.CloseIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.exists.IndicesExistsRequestBuilder; +import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder; +import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotRequestBuilder; +import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequestBuilder; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; +import org.elasticsearch.action.admin.indices.open.OpenIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.optimize.OptimizeRequestBuilder; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; +import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequestBuilder; +import org.elasticsearch.action.admin.indices.settings.UpdateSettingsRequestBuilder; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; +import org.elasticsearch.action.admin.indices.status.IndicesStatusRequestBuilder; +import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.elasticsearch.client.internal.InternalIndicesAdminClient; import org.elasticsearch.common.Nullable; diff --git a/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java b/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java index d4a2b518caa..f79c2088139 100644 --- a/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java +++ b/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java @@ -27,10 +27,10 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.bulk.BulkRequestBuilder; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; diff --git a/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java b/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java index 4935e00adbf..864a680bfa7 100644 --- a/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java +++ b/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java @@ -20,11 +20,11 @@ package org.elasticsearch.benchmark.search.child; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; -import org.elasticsearch.client.action.bulk.BulkRequestBuilder; import org.elasticsearch.common.StopWatch; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.SizeValue; diff --git a/src/test/java/org/elasticsearch/benchmark/search/facet/HistogramFacetSearchBenchmark.java b/src/test/java/org/elasticsearch/benchmark/search/facet/HistogramFacetSearchBenchmark.java index d2b33a59474..4912a791d3e 100644 --- a/src/test/java/org/elasticsearch/benchmark/search/facet/HistogramFacetSearchBenchmark.java +++ b/src/test/java/org/elasticsearch/benchmark/search/facet/HistogramFacetSearchBenchmark.java @@ -20,11 +20,11 @@ package org.elasticsearch.benchmark.search.facet; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; -import org.elasticsearch.client.action.bulk.BulkRequestBuilder; import org.elasticsearch.common.StopWatch; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.SizeValue; diff --git a/src/test/java/org/elasticsearch/benchmark/search/facet/QueryFilterFacetSearchBenchmark.java b/src/test/java/org/elasticsearch/benchmark/search/facet/QueryFilterFacetSearchBenchmark.java index 34397a4f290..0cac33d19cd 100644 --- a/src/test/java/org/elasticsearch/benchmark/search/facet/QueryFilterFacetSearchBenchmark.java +++ b/src/test/java/org/elasticsearch/benchmark/search/facet/QueryFilterFacetSearchBenchmark.java @@ -21,12 +21,12 @@ package org.elasticsearch.benchmark.search.facet; import jsr166y.ThreadLocalRandom; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; -import org.elasticsearch.client.action.bulk.BulkRequestBuilder; import org.elasticsearch.common.StopWatch; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.SizeValue; diff --git a/src/test/java/org/elasticsearch/benchmark/search/facet/TermsFacetSearchBenchmark.java b/src/test/java/org/elasticsearch/benchmark/search/facet/TermsFacetSearchBenchmark.java index 3fab24656d3..9444308f89f 100644 --- a/src/test/java/org/elasticsearch/benchmark/search/facet/TermsFacetSearchBenchmark.java +++ b/src/test/java/org/elasticsearch/benchmark/search/facet/TermsFacetSearchBenchmark.java @@ -22,12 +22,12 @@ package org.elasticsearch.benchmark.search.facet; import com.google.common.collect.Lists; import jsr166y.ThreadLocalRandom; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; -import org.elasticsearch.client.action.bulk.BulkRequestBuilder; import org.elasticsearch.common.RandomStringGenerator; import org.elasticsearch.common.StopWatch; import org.elasticsearch.common.settings.Settings; diff --git a/src/test/java/org/elasticsearch/benchmark/stress/SingleThreadBulkStress.java b/src/test/java/org/elasticsearch/benchmark/stress/SingleThreadBulkStress.java index b1c8ee1c545..e415158ad00 100644 --- a/src/test/java/org/elasticsearch/benchmark/stress/SingleThreadBulkStress.java +++ b/src/test/java/org/elasticsearch/benchmark/stress/SingleThreadBulkStress.java @@ -19,10 +19,10 @@ package org.elasticsearch.benchmark.stress; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; -import org.elasticsearch.client.action.bulk.BulkRequestBuilder; import org.elasticsearch.common.StopWatch; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.SizeValue; diff --git a/src/test/java/org/elasticsearch/test/integration/indices/analyze/AnalyzeActionTests.java b/src/test/java/org/elasticsearch/test/integration/indices/analyze/AnalyzeActionTests.java index 949ac274f56..8eadfffcc8b 100644 --- a/src/test/java/org/elasticsearch/test/integration/indices/analyze/AnalyzeActionTests.java +++ b/src/test/java/org/elasticsearch/test/integration/indices/analyze/AnalyzeActionTests.java @@ -19,9 +19,9 @@ package org.elasticsearch.test.integration.indices.analyze; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder; import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.admin.indices.analyze.AnalyzeRequestBuilder; import org.elasticsearch.test.integration.AbstractNodesTests; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; diff --git a/src/test/java/org/elasticsearch/test/integration/percolator/SimplePercolatorTests.java b/src/test/java/org/elasticsearch/test/integration/percolator/SimplePercolatorTests.java index be720b5693b..11c51abe6c6 100644 --- a/src/test/java/org/elasticsearch/test/integration/percolator/SimplePercolatorTests.java +++ b/src/test/java/org/elasticsearch/test/integration/percolator/SimplePercolatorTests.java @@ -20,11 +20,11 @@ package org.elasticsearch.test.integration.percolator; import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.bulk.BulkRequestBuilder; import org.elasticsearch.test.integration.AbstractNodesTests; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; diff --git a/src/test/java/org/elasticsearch/test/integration/readonly/ClusterAndIndexReaderOnlyTests.java b/src/test/java/org/elasticsearch/test/integration/readonly/ClusterAndIndexReaderOnlyTests.java index b3a61be9185..2b0b1d3716f 100644 --- a/src/test/java/org/elasticsearch/test/integration/readonly/ClusterAndIndexReaderOnlyTests.java +++ b/src/test/java/org/elasticsearch/test/integration/readonly/ClusterAndIndexReaderOnlyTests.java @@ -19,15 +19,15 @@ package org.elasticsearch.test.integration.readonly; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequestBuilder; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.exists.IndicesExistsResponse; +import org.elasticsearch.action.admin.indices.settings.UpdateSettingsRequestBuilder; import org.elasticsearch.action.admin.indices.settings.UpdateSettingsResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.client.action.admin.cluster.settings.ClusterUpdateSettingsRequestBuilder; -import org.elasticsearch.client.action.admin.indices.settings.UpdateSettingsRequestBuilder; -import org.elasticsearch.client.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; diff --git a/src/test/java/org/elasticsearch/test/stress/fullrestart/FullRestartStressTest.java b/src/test/java/org/elasticsearch/test/stress/fullrestart/FullRestartStressTest.java index 2a1d7783d45..a9707f8f222 100644 --- a/src/test/java/org/elasticsearch/test/stress/fullrestart/FullRestartStressTest.java +++ b/src/test/java/org/elasticsearch/test/stress/fullrestart/FullRestartStressTest.java @@ -21,10 +21,10 @@ package org.elasticsearch.test.stress.fullrestart; import jsr166y.ThreadLocalRandom; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Requests; -import org.elasticsearch.client.action.bulk.BulkRequestBuilder; import org.elasticsearch.common.UUID; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.logging.ESLogger; diff --git a/src/test/java/org/elasticsearch/test/stress/indexing/BulkIndexingStressTest.java b/src/test/java/org/elasticsearch/test/stress/indexing/BulkIndexingStressTest.java index c36f0412bfc..3bedcad4a71 100644 --- a/src/test/java/org/elasticsearch/test/stress/indexing/BulkIndexingStressTest.java +++ b/src/test/java/org/elasticsearch/test/stress/indexing/BulkIndexingStressTest.java @@ -20,9 +20,9 @@ package org.elasticsearch.test.stress.indexing; import jsr166y.ThreadLocalRandom; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; -import org.elasticsearch.client.action.bulk.BulkRequestBuilder; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.Node; diff --git a/src/test/java/org/elasticsearch/test/stress/search1/ParentChildStressTest.java b/src/test/java/org/elasticsearch/test/stress/search1/ParentChildStressTest.java index 536e760a717..85529e9295d 100644 --- a/src/test/java/org/elasticsearch/test/stress/search1/ParentChildStressTest.java +++ b/src/test/java/org/elasticsearch/test/stress/search1/ParentChildStressTest.java @@ -20,12 +20,12 @@ package org.elasticsearch.test.stress.search1; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; -import org.elasticsearch.client.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; diff --git a/src/test/java/org/elasticsearch/test/stress/search1/Search1StressTest.java b/src/test/java/org/elasticsearch/test/stress/search1/Search1StressTest.java index 15beece477a..2a7cb103e40 100644 --- a/src/test/java/org/elasticsearch/test/stress/search1/Search1StressTest.java +++ b/src/test/java/org/elasticsearch/test/stress/search1/Search1StressTest.java @@ -20,9 +20,9 @@ package org.elasticsearch.test.stress.search1; import jsr166y.ThreadLocalRandom; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.client.action.search.SearchRequestBuilder; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.ImmutableSettings; From e37c0904f0f934fcfe134222ef505976ff377727 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 15 Jan 2012 16:15:05 +0200 Subject: [PATCH 170/270] Add generic execution of APIs to Client (and indices/cluster) and allow for plugins to register custom APIs, closes #1612. --- .../SinglePingResponse.java => Action.java} | 21 +- .../elasticsearch/action/ActionModule.java | 240 ++++++++ .../elasticsearch/action/GenericAction.java | 66 ++ .../action/TransportActionModule.java | 196 ------ .../TransportActionNodeProxy.java} | 51 +- .../action/TransportActions.java | 102 ---- .../admin/cluster/ClusterAction.java} | 20 +- .../cluster/health/ClusterHealthAction.java | 45 ++ .../health/TransportClusterHealthAction.java | 3 +- .../cluster/node/info/NodesInfoAction.java | 45 ++ .../node/info/TransportNodesInfoAction.java | 3 +- .../node/restart/NodesRestartAction.java | 45 ++ .../restart/TransportNodesRestartAction.java | 3 +- .../node/shutdown/NodesShutdownAction.java | 45 ++ .../TransportNodesShutdownAction.java | 3 +- .../cluster/node/stats/NodesStatsAction.java | 45 ++ .../node/stats/TransportNodesStatsAction.java | 3 +- .../ping/broadcast/BroadcastPingRequest.java | 85 --- .../BroadcastPingRequestBuilder.java | 61 -- .../ping/broadcast/BroadcastPingResponse.java | 52 -- .../TransportBroadcastPingAction.java | 125 ---- .../IndexReplicationPingRequest.java | 69 --- .../IndexReplicationPingResponse.java | 79 --- .../replication/ReplicationPingRequest.java | 58 -- .../ReplicationPingRequestBuilder.java | 66 -- .../replication/ReplicationPingResponse.java | 67 --- .../ShardReplicationPingRequest.java | 88 --- .../TransportIndexReplicationPingAction.java | 101 ---- .../TransportReplicationPingAction.java | 86 --- .../TransportShardReplicationPingAction.java | 100 ---- .../ping/single/SinglePingRequest.java | 89 --- .../ping/single/SinglePingRequestBuilder.java | 64 -- .../single/TransportSinglePingAction.java | 85 --- .../cluster/reroute/ClusterRerouteAction.java | 45 ++ .../TransportClusterRerouteAction.java | 3 +- .../settings/ClusterUpdateSettingsAction.java | 45 ++ .../TransportClusterUpdateSettingsAction.java | 3 +- .../cluster/state/ClusterStateAction.java | 45 ++ .../state/TransportClusterStateAction.java | 3 +- .../action/admin/indices/IndicesAction.java | 39 ++ .../indices/alias/IndicesAliasesAction.java | 45 ++ .../alias/TransportIndicesAliasesAction.java | 3 +- .../admin/indices/analyze/AnalyzeAction.java | 45 ++ .../analyze/AnalyzeRequestBuilder.java | 4 + .../analyze/TransportAnalyzeAction.java | 3 +- .../cache/clear/ClearIndicesCacheAction.java | 45 ++ .../TransportClearIndicesCacheAction.java | 3 +- .../admin/indices/close/CloseIndexAction.java | 45 ++ .../indices/close/CloseIndexRequest.java | 5 + .../close/CloseIndexRequestBuilder.java | 9 + .../close/TransportCloseIndexAction.java | 5 +- .../indices/create/CreateIndexAction.java | 45 ++ .../indices/create/CreateIndexRequest.java | 5 + .../create/CreateIndexRequestBuilder.java | 9 + .../create/TransportCreateIndexAction.java | 5 +- .../indices/delete/DeleteIndexAction.java | 45 ++ .../delete/TransportDeleteIndexAction.java | 5 +- .../indices/exists/IndicesExistsAction.java | 45 ++ .../exists/TransportIndicesExistsAction.java | 5 +- .../admin/indices/flush/FlushAction.java | 45 ++ .../indices/flush/TransportFlushAction.java | 3 +- .../snapshot/GatewaySnapshotAction.java | 45 ++ .../TransportGatewaySnapshotAction.java | 3 +- .../mapping/delete/DeleteMappingAction.java | 45 ++ .../delete/TransportDeleteMappingAction.java | 5 +- .../indices/mapping/put/PutMappingAction.java | 45 ++ .../put/TransportPutMappingAction.java | 5 +- .../admin/indices/open/OpenIndexAction.java | 45 ++ .../admin/indices/open/OpenIndexRequest.java | 5 + .../indices/open/OpenIndexRequestBuilder.java | 9 + .../open/TransportOpenIndexAction.java | 5 +- .../indices/optimize/OptimizeAction.java | 45 ++ .../optimize/TransportOptimizeAction.java | 3 +- .../admin/indices/refresh/RefreshAction.java | 45 ++ .../refresh/TransportRefreshAction.java | 3 +- .../segments/IndicesSegmentsAction.java | 45 ++ .../TransportIndicesSegmentsAction.java | 3 +- .../TransportUpdateSettingsAction.java | 3 +- .../settings/UpdateSettingsAction.java | 45 ++ .../indices/stats/IndicesStatsAction.java | 45 ++ .../stats/TransportIndicesStatsAction.java | 3 +- .../indices/status/IndicesStatusAction.java | 45 ++ .../status/TransportIndicesStatusAction.java | 3 +- .../delete/DeleteIndexTemplateAction.java | 45 ++ .../DeleteIndexTemplateRequestBuilder.java | 4 + .../TransportDeleteIndexTemplateAction.java | 3 +- .../template/put/PutIndexTemplateAction.java | 45 ++ .../put/PutIndexTemplateRequestBuilder.java | 4 + .../put/TransportPutIndexTemplateAction.java | 3 +- .../query/TransportValidateQueryAction.java | 3 +- .../validate/query/ValidateQueryAction.java | 45 ++ .../bulk/BulkAction.java} | 33 +- .../action/bulk/TransportBulkAction.java | 5 +- .../action/bulk/TransportShardBulkAction.java | 3 +- .../CountAction.java} | 31 +- .../action/count/TransportCountAction.java | 3 +- .../DeleteAction.java} | 31 +- .../action/delete/DeleteRequestBuilder.java | 4 + .../action/delete/TransportDeleteAction.java | 3 +- .../deletebyquery/DeleteByQueryAction.java | 45 ++ .../TransportDeleteByQueryAction.java | 3 +- .../TransportIndexDeleteByQueryAction.java | 3 +- .../TransportShardDeleteByQueryAction.java | 3 +- .../GetAction.java} | 26 +- .../action/get/GetRequestBuilder.java | 4 + .../action/get/MultiGetAction.java | 45 ++ .../action/get/TransportGetAction.java | 3 +- .../action/get/TransportMultiGetAction.java | 5 +- .../get/TransportShardMultiGetAction.java | 3 +- .../action/index/IndexAction.java | 45 ++ .../action/index/IndexRequestBuilder.java | 4 + .../action/index/TransportIndexAction.java | 3 +- .../action/mlt/MoreLikeThisAction.java | 46 ++ .../mlt/MoreLikeThisRequestBuilder.java | 4 + .../mlt/TransportMoreLikeThisAction.java | 3 +- .../action/percolate/PercolateAction.java | 45 ++ .../percolate/PercolateRequestBuilder.java | 4 + .../percolate/TransportPercolateAction.java | 3 +- .../action/search/SearchAction.java | 45 ++ .../action/search/SearchResponse.java | 4 +- .../action/search/SearchScrollAction.java | 45 ++ .../action/search/SearchScrollRequest.java | 5 + .../search/SearchScrollRequestBuilder.java | 12 + .../action/search/TransportSearchAction.java | 3 +- .../search/TransportSearchScrollAction.java | 3 +- .../action/update/TransportUpdateAction.java | 3 +- .../action/update/UpdateAction.java | 45 ++ .../action/update/UpdateRequestBuilder.java | 4 + .../java/org/elasticsearch/client/Client.java | 39 +- .../client/ClusterAdminClient.java | 39 +- .../client/IndicesAdminClient.java | 11 +- .../org/elasticsearch/client/Requests.java | 17 - .../elasticsearch/client/node/NodeClient.java | 196 +----- .../client/node/NodeClusterAdminClient.java | 192 +----- .../client/node/NodeIndicesAdminClient.java | 352 +---------- .../client/support/AbstractClient.java | 155 ++++- .../support/AbstractClusterAdminClient.java | 134 ++++- .../support/AbstractIndicesAdminClient.java | 280 +++++++++ .../client/transport/TransportClient.java | 16 +- .../TransportClientNodesService.java | 6 +- .../action/ClientTransportActionModule.java | 162 ----- .../ClientTransportClusterHealthAction.java | 44 -- .../info/ClientTransportNodesInfoAction.java | 44 -- .../ClientTransportNodesRestartAction.java | 44 -- .../ClientTransportNodesShutdownAction.java | 44 -- .../ClientTransportNodesStatsAction.java | 44 -- .../ClientTransportBroadcastPingAction.java | 44 -- .../ClientTransportReplicationPingAction.java | 44 -- .../ClientTransportSinglePingAction.java | 44 -- .../ClientTransportClusterRerouteAction.java | 44 -- ...tTransportClusterUpdateSettingsAction.java | 43 -- .../ClientTransportClusterStateAction.java | 43 -- .../ClientTransportIndicesAliasesAction.java | 44 -- .../analyze/ClientTransportAnalyzeAction.java | 44 -- ...lientTransportClearIndicesCacheAction.java | 44 -- .../ClientTransportCloseIndexAction.java | 44 -- .../ClientTransportCreateIndexAction.java | 44 -- .../ClientTransportDeleteIndexAction.java | 44 -- .../ClientTransportIndicesExistsAction.java | 44 -- .../flush/ClientTransportFlushAction.java | 44 -- .../ClientTransportGatewaySnapshotAction.java | 44 -- .../ClientTransportDeleteMappingAction.java | 44 -- .../put/ClientTransportPutMappingAction.java | 44 -- .../open/ClientTransportOpenIndexAction.java | 44 -- .../ClientTransportOptimizeAction.java | 44 -- .../refresh/ClientTransportRefreshAction.java | 44 -- .../ClientTransportIndicesSegmentsAction.java | 43 -- .../ClientTransportUpdateSettingsAction.java | 44 -- .../ClientTransportIndicesStatsAction.java | 43 -- .../ClientTransportIndicesStatusAction.java | 44 -- ...entTransportDeleteIndexTemplateAction.java | 44 -- ...ClientTransportPutIndexTemplateAction.java | 44 -- .../ClientTransportValidateQueryAction.java | 25 - .../count/ClientTransportCountAction.java | 44 -- .../delete/ClientTransportDeleteAction.java | 44 -- .../ClientTransportDeleteByQueryAction.java | 44 -- .../action/get/ClientTransportGetAction.java | 44 -- .../get/ClientTransportMultiGetAction.java | 44 -- .../index/ClientTransportIndexAction.java | 44 -- .../ClientTransportMoreLikeThisAction.java | 44 -- .../ClientTransportPercolateAction.java | 44 -- .../search/ClientTransportSearchAction.java | 44 -- .../ClientTransportSearchScrollAction.java | 44 -- .../update/ClientTransportUpdateAction.java | 44 -- .../support/InternalTransportClient.java | 327 +--------- .../InternalTransportClusterAdminClient.java | 304 +--------- .../InternalTransportIndicesAdminClient.java | 564 +----------------- .../node/internal/InternalNode.java | 4 +- .../rest/action/RestActionModule.java | 7 - .../broadcast/RestBroadcastPingAction.java | 86 --- .../RestReplicationPingAction.java | 95 --- .../ping/single/RestSinglePingAction.java | 77 --- .../DiscoveryTransportClientTests.java | 11 - .../integration/ping/PingActionTests.java | 88 --- 194 files changed, 2982 insertions(+), 6098 deletions(-) rename src/main/java/org/elasticsearch/action/{admin/cluster/ping/single/SinglePingResponse.java => Action.java} (63%) create mode 100644 src/main/java/org/elasticsearch/action/ActionModule.java create mode 100644 src/main/java/org/elasticsearch/action/GenericAction.java delete mode 100644 src/main/java/org/elasticsearch/action/TransportActionModule.java rename src/main/java/org/elasticsearch/{client/transport/action/support/BaseClientTransportAction.java => action/TransportActionNodeProxy.java} (54%) delete mode 100644 src/main/java/org/elasticsearch/action/TransportActions.java rename src/main/java/org/elasticsearch/{client/transport/action/ClientTransportAction.java => action/admin/cluster/ClusterAction.java} (59%) create mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/node/restart/NodesRestartAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesShutdownAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsAction.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/BroadcastPingRequest.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/BroadcastPingRequestBuilder.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/BroadcastPingResponse.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/TransportBroadcastPingAction.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/IndexReplicationPingRequest.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/IndexReplicationPingResponse.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ReplicationPingRequest.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ReplicationPingRequestBuilder.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ReplicationPingResponse.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ShardReplicationPingRequest.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportIndexReplicationPingAction.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportReplicationPingAction.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportShardReplicationPingAction.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/single/SinglePingRequest.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/single/SinglePingRequestBuilder.java delete mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/ping/single/TransportSinglePingAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/IndicesAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/flush/FlushAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/gateway/snapshot/GatewaySnapshotAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/DeleteMappingAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/settings/UpdateSettingsAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/status/IndicesStatusAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateAction.java create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryAction.java rename src/main/java/org/elasticsearch/{client/transport/action/bulk/ClientTransportBulkAction.java => action/bulk/BulkAction.java} (54%) rename src/main/java/org/elasticsearch/action/{admin/cluster/ping/broadcast/BroadcastShardPingRequest.java => count/CountAction.java} (55%) rename src/main/java/org/elasticsearch/action/{admin/cluster/ping/broadcast/BroadcastShardPingResponse.java => delete/DeleteAction.java} (55%) create mode 100644 src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryAction.java rename src/main/java/org/elasticsearch/action/{admin/cluster/ping/replication/ShardReplicationPingResponse.java => get/GetAction.java} (60%) create mode 100644 src/main/java/org/elasticsearch/action/get/MultiGetAction.java create mode 100644 src/main/java/org/elasticsearch/action/index/IndexAction.java create mode 100644 src/main/java/org/elasticsearch/action/mlt/MoreLikeThisAction.java create mode 100644 src/main/java/org/elasticsearch/action/percolate/PercolateAction.java create mode 100644 src/main/java/org/elasticsearch/action/search/SearchAction.java create mode 100644 src/main/java/org/elasticsearch/action/search/SearchScrollAction.java create mode 100644 src/main/java/org/elasticsearch/action/update/UpdateAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/cluster/health/ClientTransportClusterHealthAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/info/ClientTransportNodesInfoAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/restart/ClientTransportNodesRestartAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/shutdown/ClientTransportNodesShutdownAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/stats/ClientTransportNodesStatsAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/cluster/ping/broadcast/ClientTransportBroadcastPingAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/cluster/ping/replication/ClientTransportReplicationPingAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/cluster/ping/single/ClientTransportSinglePingAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/cluster/reroute/ClientTransportClusterRerouteAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/cluster/settings/ClientTransportClusterUpdateSettingsAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/cluster/state/ClientTransportClusterStateAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/alias/ClientTransportIndicesAliasesAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/analyze/ClientTransportAnalyzeAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/cache/clear/ClientTransportClearIndicesCacheAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/close/ClientTransportCloseIndexAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/create/ClientTransportCreateIndexAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/delete/ClientTransportDeleteIndexAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/exists/ClientTransportIndicesExistsAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/flush/ClientTransportFlushAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/gateway/snapshot/ClientTransportGatewaySnapshotAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/mapping/delete/ClientTransportDeleteMappingAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/mapping/put/ClientTransportPutMappingAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/open/ClientTransportOpenIndexAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/optimize/ClientTransportOptimizeAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/refresh/ClientTransportRefreshAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/segments/ClientTransportIndicesSegmentsAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/settings/ClientTransportUpdateSettingsAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/stats/ClientTransportIndicesStatsAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/status/ClientTransportIndicesStatusAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/delete/ClientTransportDeleteIndexTemplateAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/put/ClientTransportPutIndexTemplateAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/admin/indices/validate/query/ClientTransportValidateQueryAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/count/ClientTransportCountAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/delete/ClientTransportDeleteAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/deletebyquery/ClientTransportDeleteByQueryAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/get/ClientTransportGetAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/get/ClientTransportMultiGetAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/index/ClientTransportIndexAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/mlt/ClientTransportMoreLikeThisAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/percolate/ClientTransportPercolateAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/search/ClientTransportSearchAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/search/ClientTransportSearchScrollAction.java delete mode 100644 src/main/java/org/elasticsearch/client/transport/action/update/ClientTransportUpdateAction.java delete mode 100644 src/main/java/org/elasticsearch/rest/action/admin/cluster/ping/broadcast/RestBroadcastPingAction.java delete mode 100644 src/main/java/org/elasticsearch/rest/action/admin/cluster/ping/replication/RestReplicationPingAction.java delete mode 100644 src/main/java/org/elasticsearch/rest/action/admin/cluster/ping/single/RestSinglePingAction.java delete mode 100644 src/test/java/org/elasticsearch/test/integration/ping/PingActionTests.java diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/SinglePingResponse.java b/src/main/java/org/elasticsearch/action/Action.java similarity index 63% rename from src/main/java/org/elasticsearch/action/admin/cluster/ping/single/SinglePingResponse.java rename to src/main/java/org/elasticsearch/action/Action.java index ecc6d6c4a26..9b02edc05bf 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/SinglePingResponse.java +++ b/src/main/java/org/elasticsearch/action/Action.java @@ -17,24 +17,19 @@ * under the License. */ -package org.elasticsearch.action.admin.cluster.ping.single; +package org.elasticsearch.action; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; - -import java.io.IOException; +import org.elasticsearch.client.Client; /** - * + * Main action (used with {@link Client} API. */ -public class SinglePingResponse implements ActionResponse { +public abstract class Action> + extends GenericAction { - @Override - public void readFrom(StreamInput in) throws IOException { + protected Action(String name) { + super(name); } - @Override - public void writeTo(StreamOutput out) throws IOException { - } + public abstract RequestBuilder newRequestBuilder(Client client); } diff --git a/src/main/java/org/elasticsearch/action/ActionModule.java b/src/main/java/org/elasticsearch/action/ActionModule.java new file mode 100644 index 00000000000..6ed8a25ae89 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/ActionModule.java @@ -0,0 +1,240 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action; + +import com.google.common.collect.Maps; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; +import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; +import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartAction; +import org.elasticsearch.action.admin.cluster.node.restart.TransportNodesRestartAction; +import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownAction; +import org.elasticsearch.action.admin.cluster.node.shutdown.TransportNodesShutdownAction; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction; +import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction; +import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteAction; +import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsAction; +import org.elasticsearch.action.admin.cluster.settings.TransportClusterUpdateSettingsAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.TransportClusterStateAction; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; +import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; +import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheAction; +import org.elasticsearch.action.admin.indices.cache.clear.TransportClearIndicesCacheAction; +import org.elasticsearch.action.admin.indices.close.CloseIndexAction; +import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; +import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; +import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction; +import org.elasticsearch.action.admin.indices.exists.IndicesExistsAction; +import org.elasticsearch.action.admin.indices.exists.TransportIndicesExistsAction; +import org.elasticsearch.action.admin.indices.flush.FlushAction; +import org.elasticsearch.action.admin.indices.flush.TransportFlushAction; +import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotAction; +import org.elasticsearch.action.admin.indices.gateway.snapshot.TransportGatewaySnapshotAction; +import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingAction; +import org.elasticsearch.action.admin.indices.mapping.delete.TransportDeleteMappingAction; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; +import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction; +import org.elasticsearch.action.admin.indices.open.OpenIndexAction; +import org.elasticsearch.action.admin.indices.open.TransportOpenIndexAction; +import org.elasticsearch.action.admin.indices.optimize.OptimizeAction; +import org.elasticsearch.action.admin.indices.optimize.TransportOptimizeAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.TransportRefreshAction; +import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsAction; +import org.elasticsearch.action.admin.indices.segments.TransportIndicesSegmentsAction; +import org.elasticsearch.action.admin.indices.settings.TransportUpdateSettingsAction; +import org.elasticsearch.action.admin.indices.settings.UpdateSettingsAction; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; +import org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction; +import org.elasticsearch.action.admin.indices.status.IndicesStatusAction; +import org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction; +import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.TransportPutIndexTemplateAction; +import org.elasticsearch.action.admin.indices.validate.query.TransportValidateQueryAction; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction; +import org.elasticsearch.action.bulk.BulkAction; +import org.elasticsearch.action.bulk.TransportBulkAction; +import org.elasticsearch.action.bulk.TransportShardBulkAction; +import org.elasticsearch.action.count.CountAction; +import org.elasticsearch.action.count.TransportCountAction; +import org.elasticsearch.action.delete.DeleteAction; +import org.elasticsearch.action.delete.TransportDeleteAction; +import org.elasticsearch.action.delete.index.TransportIndexDeleteAction; +import org.elasticsearch.action.delete.index.TransportShardDeleteAction; +import org.elasticsearch.action.deletebyquery.DeleteByQueryAction; +import org.elasticsearch.action.deletebyquery.TransportDeleteByQueryAction; +import org.elasticsearch.action.deletebyquery.TransportIndexDeleteByQueryAction; +import org.elasticsearch.action.deletebyquery.TransportShardDeleteByQueryAction; +import org.elasticsearch.action.get.*; +import org.elasticsearch.action.index.IndexAction; +import org.elasticsearch.action.index.TransportIndexAction; +import org.elasticsearch.action.mlt.MoreLikeThisAction; +import org.elasticsearch.action.mlt.TransportMoreLikeThisAction; +import org.elasticsearch.action.percolate.PercolateAction; +import org.elasticsearch.action.percolate.TransportPercolateAction; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchScrollAction; +import org.elasticsearch.action.search.TransportSearchAction; +import org.elasticsearch.action.search.TransportSearchScrollAction; +import org.elasticsearch.action.search.type.*; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.action.update.TransportUpdateAction; +import org.elasticsearch.action.update.UpdateAction; +import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.inject.multibindings.MapBinder; + +import java.util.Map; + +/** + * + */ +public class ActionModule extends AbstractModule { + + private final Map actions = Maps.newHashMap(); + + static class ActionEntry { + public final GenericAction action; + public final Class> transportAction; + public final Class[] supportTransportActions; + + ActionEntry(GenericAction action, Class> transportAction, Class... supportTransportActions) { + this.action = action; + this.transportAction = transportAction; + this.supportTransportActions = supportTransportActions; + } + + + } + + private final boolean proxy; + + public ActionModule(boolean proxy) { + this.proxy = proxy; + } + + /** + * Registers an action. + * + * @param action The action type. + * @param transportAction The transport action implementing the actual action. + * @param supportTransportActions Any support actions that are needed by the transport action. + * @param The request type. + * @param The response type. + */ + public void registerAction(GenericAction action, Class> transportAction, Class... supportTransportActions) { + actions.put(action.name(), new ActionEntry(action, transportAction, supportTransportActions)); + } + + @Override + protected void configure() { + + registerAction(NodesInfoAction.INSTANCE, TransportNodesInfoAction.class); + registerAction(NodesStatsAction.INSTANCE, TransportNodesStatsAction.class); + registerAction(NodesShutdownAction.INSTANCE, TransportNodesShutdownAction.class); + registerAction(NodesRestartAction.INSTANCE, TransportNodesRestartAction.class); + + registerAction(ClusterStateAction.INSTANCE, TransportClusterStateAction.class); + registerAction(ClusterHealthAction.INSTANCE, TransportClusterHealthAction.class); + registerAction(ClusterUpdateSettingsAction.INSTANCE, TransportClusterUpdateSettingsAction.class); + registerAction(ClusterRerouteAction.INSTANCE, TransportClusterRerouteAction.class); + + registerAction(IndicesStatsAction.INSTANCE, TransportIndicesStatsAction.class); + registerAction(IndicesStatusAction.INSTANCE, TransportIndicesStatusAction.class); + registerAction(IndicesSegmentsAction.INSTANCE, TransportIndicesSegmentsAction.class); + registerAction(CreateIndexAction.INSTANCE, TransportCreateIndexAction.class); + registerAction(DeleteIndexAction.INSTANCE, TransportDeleteIndexAction.class); + registerAction(OpenIndexAction.INSTANCE, TransportOpenIndexAction.class); + registerAction(CloseIndexAction.INSTANCE, TransportCloseIndexAction.class); + registerAction(IndicesExistsAction.INSTANCE, TransportIndicesExistsAction.class); + registerAction(PutMappingAction.INSTANCE, TransportPutMappingAction.class); + registerAction(DeleteMappingAction.INSTANCE, TransportDeleteMappingAction.class); + registerAction(IndicesAliasesAction.INSTANCE, TransportIndicesAliasesAction.class); + registerAction(UpdateSettingsAction.INSTANCE, TransportUpdateSettingsAction.class); + registerAction(AnalyzeAction.INSTANCE, TransportAnalyzeAction.class); + registerAction(PutIndexTemplateAction.INSTANCE, TransportPutIndexTemplateAction.class); + registerAction(DeleteIndexTemplateAction.INSTANCE, TransportDeleteIndexTemplateAction.class); + registerAction(ValidateQueryAction.INSTANCE, TransportValidateQueryAction.class); + registerAction(GatewaySnapshotAction.INSTANCE, TransportGatewaySnapshotAction.class); + registerAction(RefreshAction.INSTANCE, TransportRefreshAction.class); + registerAction(FlushAction.INSTANCE, TransportFlushAction.class); + registerAction(OptimizeAction.INSTANCE, TransportOptimizeAction.class); + registerAction(ClearIndicesCacheAction.INSTANCE, TransportClearIndicesCacheAction.class); + + registerAction(IndexAction.INSTANCE, TransportIndexAction.class); + registerAction(GetAction.INSTANCE, TransportGetAction.class); + registerAction(DeleteAction.INSTANCE, TransportDeleteAction.class, + TransportIndexDeleteAction.class, TransportShardDeleteAction.class); + registerAction(CountAction.INSTANCE, TransportCountAction.class); + registerAction(UpdateAction.INSTANCE, TransportUpdateAction.class); + registerAction(MultiGetAction.INSTANCE, TransportMultiGetAction.class, + TransportShardMultiGetAction.class); + registerAction(BulkAction.INSTANCE, TransportBulkAction.class, + TransportShardBulkAction.class); + registerAction(DeleteByQueryAction.INSTANCE, TransportDeleteByQueryAction.class, + TransportIndexDeleteByQueryAction.class, TransportShardDeleteByQueryAction.class); + registerAction(SearchAction.INSTANCE, TransportSearchAction.class, + TransportSearchCache.class, + TransportSearchDfsQueryThenFetchAction.class, + TransportSearchQueryThenFetchAction.class, + TransportSearchDfsQueryAndFetchAction.class, + TransportSearchQueryAndFetchAction.class, + TransportSearchScanAction.class + ); + registerAction(SearchScrollAction.INSTANCE, TransportSearchScrollAction.class, + TransportSearchScrollScanAction.class, + TransportSearchScrollQueryThenFetchAction.class, + TransportSearchScrollQueryAndFetchAction.class + ); + registerAction(MoreLikeThisAction.INSTANCE, TransportMoreLikeThisAction.class); + registerAction(PercolateAction.INSTANCE, TransportPercolateAction.class); + + // register Name -> GenericAction Map that can be injected to instances. + MapBinder actionsBinder + = MapBinder.newMapBinder(binder(), String.class, GenericAction.class); + + for (Map.Entry entry : actions.entrySet()) { + actionsBinder.addBinding(entry.getKey()).toInstance(entry.getValue().action); + } + + // register GenericAction -> transportAction Map that can be injected to instances. + // also register any supporting classes + if (!proxy) { + MapBinder transportActionsBinder + = MapBinder.newMapBinder(binder(), GenericAction.class, TransportAction.class); + for (Map.Entry entry : actions.entrySet()) { + // bind the action as eager singleton, so the map binder one will reuse it + bind(entry.getValue().transportAction).asEagerSingleton(); + transportActionsBinder.addBinding(entry.getValue().action).to(entry.getValue().transportAction).asEagerSingleton(); + for (Class supportAction : entry.getValue().supportTransportActions) { + bind(supportAction).asEagerSingleton(); + } + } + } + } +} diff --git a/src/main/java/org/elasticsearch/action/GenericAction.java b/src/main/java/org/elasticsearch/action/GenericAction.java new file mode 100644 index 00000000000..4acd3d2da0f --- /dev/null +++ b/src/main/java/org/elasticsearch/action/GenericAction.java @@ -0,0 +1,66 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action; + +import org.elasticsearch.transport.TransportRequestOptions; + +/** + * A generic action. Should strive to make it a singleton. + */ +public abstract class GenericAction { + + private final String name; + + /** + * @param name The name of the action, must be unique across actions. + */ + protected GenericAction(String name) { + this.name = name; + } + + /** + * The name of the action. Must be unique across actions. + */ + public String name() { + return this.name; + } + + /** + * Creates a new response instance. + */ + public abstract Response newResponse(); + + /** + * Optional request options for the action. + */ + public TransportRequestOptions options() { + return TransportRequestOptions.EMPTY; + } + + @Override + public boolean equals(Object o) { + return name.equals(((GenericAction) o).name()); + } + + @Override + public int hashCode() { + return name.hashCode(); + } +} diff --git a/src/main/java/org/elasticsearch/action/TransportActionModule.java b/src/main/java/org/elasticsearch/action/TransportActionModule.java deleted file mode 100644 index 4f9d308930f..00000000000 --- a/src/main/java/org/elasticsearch/action/TransportActionModule.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action; - -import com.google.common.collect.Maps; -import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction; -import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; -import org.elasticsearch.action.admin.cluster.node.restart.TransportNodesRestartAction; -import org.elasticsearch.action.admin.cluster.node.shutdown.TransportNodesShutdownAction; -import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction; -import org.elasticsearch.action.admin.cluster.ping.broadcast.TransportBroadcastPingAction; -import org.elasticsearch.action.admin.cluster.ping.replication.TransportIndexReplicationPingAction; -import org.elasticsearch.action.admin.cluster.ping.replication.TransportReplicationPingAction; -import org.elasticsearch.action.admin.cluster.ping.replication.TransportShardReplicationPingAction; -import org.elasticsearch.action.admin.cluster.ping.single.TransportSinglePingAction; -import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction; -import org.elasticsearch.action.admin.cluster.settings.TransportClusterUpdateSettingsAction; -import org.elasticsearch.action.admin.cluster.state.TransportClusterStateAction; -import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction; -import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction; -import org.elasticsearch.action.admin.indices.cache.clear.TransportClearIndicesCacheAction; -import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction; -import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; -import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction; -import org.elasticsearch.action.admin.indices.exists.TransportIndicesExistsAction; -import org.elasticsearch.action.admin.indices.flush.TransportFlushAction; -import org.elasticsearch.action.admin.indices.gateway.snapshot.TransportGatewaySnapshotAction; -import org.elasticsearch.action.admin.indices.mapping.delete.TransportDeleteMappingAction; -import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction; -import org.elasticsearch.action.admin.indices.open.TransportOpenIndexAction; -import org.elasticsearch.action.admin.indices.optimize.TransportOptimizeAction; -import org.elasticsearch.action.admin.indices.refresh.TransportRefreshAction; -import org.elasticsearch.action.admin.indices.segments.TransportIndicesSegmentsAction; -import org.elasticsearch.action.admin.indices.settings.TransportUpdateSettingsAction; -import org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction; -import org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction; -import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteIndexTemplateAction; -import org.elasticsearch.action.admin.indices.template.put.TransportPutIndexTemplateAction; -import org.elasticsearch.action.admin.indices.validate.query.TransportValidateQueryAction; -import org.elasticsearch.action.bulk.TransportBulkAction; -import org.elasticsearch.action.bulk.TransportShardBulkAction; -import org.elasticsearch.action.count.TransportCountAction; -import org.elasticsearch.action.delete.TransportDeleteAction; -import org.elasticsearch.action.delete.index.TransportIndexDeleteAction; -import org.elasticsearch.action.delete.index.TransportShardDeleteAction; -import org.elasticsearch.action.deletebyquery.TransportDeleteByQueryAction; -import org.elasticsearch.action.deletebyquery.TransportIndexDeleteByQueryAction; -import org.elasticsearch.action.deletebyquery.TransportShardDeleteByQueryAction; -import org.elasticsearch.action.get.TransportGetAction; -import org.elasticsearch.action.get.TransportMultiGetAction; -import org.elasticsearch.action.get.TransportShardMultiGetAction; -import org.elasticsearch.action.index.TransportIndexAction; -import org.elasticsearch.action.mlt.TransportMoreLikeThisAction; -import org.elasticsearch.action.percolate.TransportPercolateAction; -import org.elasticsearch.action.search.TransportSearchAction; -import org.elasticsearch.action.search.TransportSearchScrollAction; -import org.elasticsearch.action.search.type.*; -import org.elasticsearch.action.support.TransportAction; -import org.elasticsearch.action.update.TransportUpdateAction; -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.inject.multibindings.MapBinder; - -import java.util.Map; - -/** - * - */ -public class TransportActionModule extends AbstractModule { - - private final Map actions = Maps.newHashMap(); - - static class ActionEntry { - public final String name; - public final Class action; - public final Class[] supportActions; - - ActionEntry(String name, Class action, Class... supportActions) { - this.name = name; - this.action = action; - this.supportActions = supportActions; - } - - - } - - public TransportActionModule() { - } - - /** - * Registers a custom action under the provided action name, the actual action implementation, and - * any supported actions (bind as singletons). - * - * @param actionName The action name - * @param action The action itself - * @param supportActions Support actions. - */ - public void registerAction(String actionName, Class action, Class... supportActions) { - actions.put(actionName, new ActionEntry(actionName, action, supportActions)); - } - - @Override - protected void configure() { - - registerAction(TransportActions.Admin.Cluster.Node.INFO, TransportNodesInfoAction.class); - registerAction(TransportActions.Admin.Cluster.Node.STATS, TransportNodesStatsAction.class); - registerAction(TransportActions.Admin.Cluster.Node.SHUTDOWN, TransportNodesShutdownAction.class); - registerAction(TransportActions.Admin.Cluster.Node.RESTART, TransportNodesRestartAction.class); - - registerAction(TransportActions.Admin.Cluster.STATE, TransportClusterStateAction.class); - registerAction(TransportActions.Admin.Cluster.HEALTH, TransportClusterHealthAction.class); - registerAction(TransportActions.Admin.Cluster.UPDATE_SETTINGS, TransportClusterUpdateSettingsAction.class); - registerAction(TransportActions.Admin.Cluster.REROUTE, TransportClusterRerouteAction.class); - - registerAction(TransportActions.Admin.Cluster.Ping.SINGLE, TransportSinglePingAction.class); - registerAction(TransportActions.Admin.Cluster.Ping.BROADCAST, TransportBroadcastPingAction.class); - registerAction(TransportActions.Admin.Cluster.Ping.REPLICATION, TransportReplicationPingAction.class, - TransportIndexReplicationPingAction.class, TransportShardReplicationPingAction.class); - - registerAction(TransportActions.Admin.Indices.STATS, TransportIndicesStatsAction.class); - registerAction(TransportActions.Admin.Indices.STATUS, TransportIndicesStatusAction.class); - registerAction(TransportActions.Admin.Indices.SEGMENTS, TransportIndicesSegmentsAction.class); - registerAction(TransportActions.Admin.Indices.CREATE, TransportCreateIndexAction.class); - registerAction(TransportActions.Admin.Indices.DELETE, TransportDeleteIndexAction.class); - registerAction(TransportActions.Admin.Indices.OPEN, TransportOpenIndexAction.class); - registerAction(TransportActions.Admin.Indices.CLOSE, TransportCloseIndexAction.class); - registerAction(TransportActions.Admin.Indices.EXISTS, TransportIndicesExistsAction.class); - registerAction(TransportActions.Admin.Indices.Mapping.PUT, TransportPutMappingAction.class); - registerAction(TransportActions.Admin.Indices.Mapping.DELETE, TransportDeleteMappingAction.class); - registerAction(TransportActions.Admin.Indices.ALIASES, TransportIndicesAliasesAction.class); - registerAction(TransportActions.Admin.Indices.UPDATE_SETTINGS, TransportUpdateSettingsAction.class); - registerAction(TransportActions.Admin.Indices.ANALYZE, TransportAnalyzeAction.class); - registerAction(TransportActions.Admin.Indices.Template.PUT, TransportPutIndexTemplateAction.class); - registerAction(TransportActions.Admin.Indices.Template.DELETE, TransportDeleteIndexTemplateAction.class); - registerAction(TransportActions.Admin.Indices.Validate.QUERY, TransportValidateQueryAction.class); - registerAction(TransportActions.Admin.Indices.Gateway.SNAPSHOT, TransportGatewaySnapshotAction.class); - registerAction(TransportActions.Admin.Indices.REFRESH, TransportRefreshAction.class); - registerAction(TransportActions.Admin.Indices.FLUSH, TransportFlushAction.class); - registerAction(TransportActions.Admin.Indices.OPTIMIZE, TransportOptimizeAction.class); - registerAction(TransportActions.Admin.Indices.Cache.CLEAR, TransportClearIndicesCacheAction.class); - - registerAction(TransportActions.INDEX, TransportIndexAction.class); - registerAction(TransportActions.GET, TransportGetAction.class); - registerAction(TransportActions.DELETE, TransportDeleteAction.class, - TransportIndexDeleteAction.class, TransportShardDeleteAction.class); - registerAction(TransportActions.COUNT, TransportCountAction.class); - registerAction(TransportActions.UPDATE, TransportUpdateAction.class); - registerAction(TransportActions.MULTI_GET, TransportMultiGetAction.class, - TransportShardMultiGetAction.class); - registerAction(TransportActions.BULK, TransportBulkAction.class, - TransportShardBulkAction.class); - registerAction(TransportActions.DELETE_BY_QUERY, TransportDeleteByQueryAction.class, - TransportIndexDeleteByQueryAction.class, TransportShardDeleteByQueryAction.class); - registerAction(TransportActions.SEARCH, TransportSearchAction.class, - TransportSearchCache.class, - TransportSearchDfsQueryThenFetchAction.class, - TransportSearchQueryThenFetchAction.class, - TransportSearchDfsQueryAndFetchAction.class, - TransportSearchQueryAndFetchAction.class, - TransportSearchScanAction.class - ); - registerAction(TransportActions.SEARCH_SCROLL, TransportSearchScrollAction.class, - TransportSearchScrollScanAction.class, - TransportSearchScrollQueryThenFetchAction.class, - TransportSearchScrollQueryAndFetchAction.class - ); - registerAction(TransportActions.MORE_LIKE_THIS, TransportMoreLikeThisAction.class); - registerAction(TransportActions.PERCOLATE, TransportPercolateAction.class); - - MapBinder actionsBinder - = MapBinder.newMapBinder(binder(), String.class, TransportAction.class); - - for (Map.Entry entry : actions.entrySet()) { - actionsBinder.addBinding(entry.getKey()).to(entry.getValue().action).asEagerSingleton(); - for (Class supportAction : entry.getValue().supportActions) { - bind(supportAction).asEagerSingleton(); - } - } - } -} diff --git a/src/main/java/org/elasticsearch/client/transport/action/support/BaseClientTransportAction.java b/src/main/java/org/elasticsearch/action/TransportActionNodeProxy.java similarity index 54% rename from src/main/java/org/elasticsearch/client/transport/action/support/BaseClientTransportAction.java rename to src/main/java/org/elasticsearch/action/TransportActionNodeProxy.java index d81315d41e6..e474082f5fd 100644 --- a/src/main/java/org/elasticsearch/client/transport/action/support/BaseClientTransportAction.java +++ b/src/main/java/org/elasticsearch/action/TransportActionNodeProxy.java @@ -17,51 +17,34 @@ * under the License. */ -package org.elasticsearch.client.transport.action.support; +package org.elasticsearch.action; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.ElasticSearchIllegalArgumentException; -import org.elasticsearch.ElasticSearchIllegalStateException; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.client.transport.action.ClientTransportAction; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.inject.Inject; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.BaseTransportResponseHandler; import org.elasticsearch.transport.TransportException; -import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; -import java.lang.reflect.Constructor; - import static org.elasticsearch.action.support.PlainActionFuture.newFuture; /** - * + * A generic proxy that will execute the given action against a specific node. */ -public abstract class BaseClientTransportAction extends AbstractComponent implements ClientTransportAction { +public class TransportActionNodeProxy { protected final TransportService transportService; - private final Constructor responseConstructor; + private final GenericAction action; - protected BaseClientTransportAction(Settings settings, TransportService transportService, Class type) { - super(settings); + @Inject + public TransportActionNodeProxy(GenericAction action, TransportService transportService) { + this.action = action; this.transportService = transportService; - try { - this.responseConstructor = type.getDeclaredConstructor(); - } catch (NoSuchMethodException e) { - throw new ElasticSearchIllegalArgumentException("No default constructor is declared for [" + type.getName() + "]"); - } - responseConstructor.setAccessible(true); } - @Override public ActionFuture execute(DiscoveryNode node, Request request) throws ElasticSearchException { PlainActionFuture future = newFuture(); request.listenerThreaded(false); @@ -69,12 +52,11 @@ public abstract class BaseClientTransportAction listener) { - transportService.sendRequest(node, action(), request, options(), new BaseTransportResponseHandler() { + transportService.sendRequest(node, action.name(), request, action.options(), new BaseTransportResponseHandler() { @Override public Response newInstance() { - return BaseClientTransportAction.this.newInstance(); + return action.newResponse(); } @Override @@ -97,17 +79,4 @@ public abstract class BaseClientTransportAction { +public abstract class ClusterAction> + extends GenericAction { - ActionFuture execute(DiscoveryNode node, Request request) throws ElasticSearchException; + protected ClusterAction(String name) { + super(name); + } - void execute(DiscoveryNode node, Request request, ActionListener listener); + public abstract RequestBuilder newRequestBuilder(ClusterAdminClient client); } diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthAction.java new file mode 100644 index 00000000000..837ca665cde --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.health; + +import org.elasticsearch.action.admin.cluster.ClusterAction; +import org.elasticsearch.client.ClusterAdminClient; + +/** + */ +public class ClusterHealthAction extends ClusterAction { + + public static final ClusterHealthAction INSTANCE = new ClusterHealthAction(); + public static final String NAME = "cluster/health"; + + private ClusterHealthAction() { + super(NAME); + } + + @Override + public ClusterHealthResponse newResponse() { + return new ClusterHealthResponse(); + } + + @Override + public ClusterHealthRequestBuilder newRequestBuilder(ClusterAdminClient client) { + return new ClusterHealthRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java index 16abebf050d..5090aa77690 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.admin.cluster.health; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; @@ -58,7 +57,7 @@ public class TransportClusterHealthAction extends TransportMasterNodeOperationAc @Override protected String transportAction() { - return TransportActions.Admin.Cluster.HEALTH; + return ClusterHealthAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoAction.java new file mode 100644 index 00000000000..5beba9e1fbb --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.node.info; + +import org.elasticsearch.action.admin.cluster.ClusterAction; +import org.elasticsearch.client.ClusterAdminClient; + +/** + */ +public class NodesInfoAction extends ClusterAction { + + public static final NodesInfoAction INSTANCE = new NodesInfoAction(); + public static final String NAME = "cluster/nodes/info"; + + private NodesInfoAction() { + super(NAME); + } + + @Override + public NodesInfoResponse newResponse() { + return new NodesInfoResponse(); + } + + @Override + public NodesInfoRequestBuilder newRequestBuilder(ClusterAdminClient client) { + return new NodesInfoRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java index b5f6d8a1341..c94c3be7ef9 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.admin.cluster.node.info; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.nodes.NodeOperationRequest; import org.elasticsearch.action.support.nodes.TransportNodesOperationAction; import org.elasticsearch.cluster.ClusterName; @@ -60,7 +59,7 @@ public class TransportNodesInfoAction extends TransportNodesOperationAction { + + public static final NodesRestartAction INSTANCE = new NodesRestartAction(); + public static final String NAME = "cluster/nodes/restart"; + + private NodesRestartAction() { + super(NAME); + } + + @Override + public NodesRestartResponse newResponse() { + return new NodesRestartResponse(); + } + + @Override + public NodesRestartRequestBuilder newRequestBuilder(ClusterAdminClient client) { + return new NodesRestartRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/restart/TransportNodesRestartAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/restart/TransportNodesRestartAction.java index 8c8cfcc1bf0..2a57d682d49 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/restart/TransportNodesRestartAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/restart/TransportNodesRestartAction.java @@ -22,7 +22,6 @@ package org.elasticsearch.action.admin.cluster.node.restart; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ElasticSearchIllegalStateException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.nodes.NodeOperationRequest; import org.elasticsearch.action.support.nodes.TransportNodesOperationAction; import org.elasticsearch.cluster.ClusterName; @@ -76,7 +75,7 @@ public class TransportNodesRestartAction extends TransportNodesOperationAction { + + public static final NodesShutdownAction INSTANCE = new NodesShutdownAction(); + public static final String NAME = "cluster/nodes/shutdown"; + + private NodesShutdownAction() { + super(NAME); + } + + @Override + public NodesShutdownResponse newResponse() { + return new NodesShutdownResponse(); + } + + @Override + public NodesShutdownRequestBuilder newRequestBuilder(ClusterAdminClient client) { + return new NodesShutdownRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportNodesShutdownAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportNodesShutdownAction.java index 148766484b2..7f8c381d291 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportNodesShutdownAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportNodesShutdownAction.java @@ -22,7 +22,6 @@ package org.elasticsearch.action.admin.cluster.node.shutdown; import com.google.common.collect.Sets; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ElasticSearchIllegalStateException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; @@ -75,7 +74,7 @@ public class TransportNodesShutdownAction extends TransportMasterNodeOperationAc @Override protected String transportAction() { - return TransportActions.Admin.Cluster.Node.SHUTDOWN; + return NodesShutdownAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsAction.java new file mode 100644 index 00000000000..d73430b83a8 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.node.stats; + +import org.elasticsearch.action.admin.cluster.ClusterAction; +import org.elasticsearch.client.ClusterAdminClient; + +/** + */ +public class NodesStatsAction extends ClusterAction { + + public static final NodesStatsAction INSTANCE = new NodesStatsAction(); + public static final String NAME = "cluster/nodes/stats"; + + private NodesStatsAction() { + super(NAME); + } + + @Override + public NodesStatsResponse newResponse() { + return new NodesStatsResponse(); + } + + @Override + public NodesStatsRequestBuilder newRequestBuilder(ClusterAdminClient client) { + return new NodesStatsRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index e93f595544f..8019a47331f 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -21,7 +21,6 @@ package org.elasticsearch.action.admin.cluster.node.stats; import com.google.common.collect.Lists; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.nodes.NodeOperationRequest; import org.elasticsearch.action.support.nodes.TransportNodesOperationAction; import org.elasticsearch.cluster.ClusterName; @@ -60,7 +59,7 @@ public class TransportNodesStatsAction extends TransportNodesOperationAction { - - public BroadcastPingRequestBuilder(ClusterAdminClient clusterClient) { - super(clusterClient, new BroadcastPingRequest()); - } - - public BroadcastPingRequestBuilder setIndices(String... indices) { - request.indices(indices); - return this; - } - - - public BroadcastPingRequestBuilder setOperationThreading(BroadcastOperationThreading operationThreading) { - request.operationThreading(operationThreading); - return this; - } - - public BroadcastPingRequestBuilder setListenerThreaded(boolean threadedListener) { - request.listenerThreaded(threadedListener); - return this; - } - - public BroadcastPingRequestBuilder setQueryHint(String queryHint) { - request.queryHint(queryHint); - return this; - } - - @Override - protected void doExecute(ActionListener listener) { - client.ping(request, listener); - } -} diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/BroadcastPingResponse.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/BroadcastPingResponse.java deleted file mode 100644 index 4bbf19f4df8..00000000000 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/BroadcastPingResponse.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.ping.broadcast; - -import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.support.broadcast.BroadcastOperationResponse; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; - -import java.io.IOException; -import java.util.List; - -/** - * - */ -public class BroadcastPingResponse extends BroadcastOperationResponse { - - BroadcastPingResponse() { - - } - - public BroadcastPingResponse(int totalShards, int successfulShards, int failedShards, List shardFailures) { - super(totalShards, successfulShards, failedShards, shardFailures); - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/TransportBroadcastPingAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/TransportBroadcastPingAction.java deleted file mode 100644 index 9f869cd295b..00000000000 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/broadcast/TransportBroadcastPingAction.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.ping.broadcast; - -import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.support.DefaultShardOperationFailedException; -import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; -import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.routing.GroupShardsIterator; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; - -import java.util.List; -import java.util.concurrent.atomic.AtomicReferenceArray; - -import static com.google.common.collect.Lists.newArrayList; - -/** - * - */ -public class TransportBroadcastPingAction extends TransportBroadcastOperationAction { - - @Inject - public TransportBroadcastPingAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService) { - super(settings, threadPool, clusterService, transportService); - } - - @Override - protected String executor() { - return ThreadPool.Names.CACHED; - } - - @Override - protected String transportAction() { - return TransportActions.Admin.Cluster.Ping.BROADCAST; - } - - @Override - protected BroadcastPingRequest newRequest() { - return new BroadcastPingRequest(); - } - - @Override - protected GroupShardsIterator shards(ClusterState clusterState, BroadcastPingRequest request, String[] concreteIndices) { - return clusterService.operationRouting().searchShards(clusterState, request.indices(), concreteIndices, request.queryHint(), null, null); - } - - @Override - protected ClusterBlockException checkGlobalBlock(ClusterState state, BroadcastPingRequest request) { - return state.blocks().globalBlockedException(ClusterBlockLevel.READ); - } - - @Override - protected ClusterBlockException checkRequestBlock(ClusterState state, BroadcastPingRequest request, String[] concreteIndices) { - return null; - } - - @Override - protected BroadcastPingResponse newResponse(BroadcastPingRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) { - int successfulShards = 0; - int failedShards = 0; - List shardFailures = null; - for (int i = 0; i < shardsResponses.length(); i++) { - Object shardResponse = shardsResponses.get(i); - if (shardResponse == null) { - failedShards++; - } else if (shardResponse instanceof BroadcastShardOperationFailedException) { - failedShards++; - if (shardFailures == null) { - shardFailures = newArrayList(); - } - shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse)); - } else { - successfulShards++; - } - } - return new BroadcastPingResponse(shardsResponses.length(), successfulShards, failedShards, shardFailures); - } - - @Override - protected BroadcastShardPingRequest newShardRequest() { - return new BroadcastShardPingRequest(); - } - - @Override - protected BroadcastShardPingRequest newShardRequest(ShardRouting shard, BroadcastPingRequest request) { - return new BroadcastShardPingRequest(shard.index(), shard.id()); - } - - @Override - protected BroadcastShardPingResponse newShardResponse() { - return new BroadcastShardPingResponse(); - } - - @Override - protected BroadcastShardPingResponse shardOperation(BroadcastShardPingRequest broadcastShardPingRequest) throws ElasticSearchException { - return new BroadcastShardPingResponse(); - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/IndexReplicationPingRequest.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/IndexReplicationPingRequest.java deleted file mode 100644 index d912f837fa2..00000000000 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/IndexReplicationPingRequest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.ping.replication; - -import org.elasticsearch.action.support.replication.IndexReplicationOperationRequest; -import org.elasticsearch.action.support.replication.ReplicationType; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.unit.TimeValue; - -import java.io.IOException; - -/** - * - */ -public class IndexReplicationPingRequest extends IndexReplicationOperationRequest { - - public IndexReplicationPingRequest(String index) { - this.index = index; - } - - IndexReplicationPingRequest(ReplicationPingRequest request, String index) { - this.index = index; - this.timeout = request.timeout(); - this.replicationType = request.replicationType(); - } - - IndexReplicationPingRequest() { - } - - public IndexReplicationPingRequest timeout(TimeValue timeout) { - this.timeout = timeout; - return this; - } - - /** - * The replication type to use with this operation. - */ - public IndexReplicationPingRequest replicationType(ReplicationType replicationType) { - this.replicationType = replicationType; - return this; - } - - - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - } - - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/IndexReplicationPingResponse.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/IndexReplicationPingResponse.java deleted file mode 100644 index 4f4ce5b3554..00000000000 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/IndexReplicationPingResponse.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.ping.replication; - -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; - -import java.io.IOException; - -/** - * - */ -public class IndexReplicationPingResponse implements ActionResponse, Streamable { - - private String index; - - private int successfulShards; - - private int failedShards; - - IndexReplicationPingResponse(String index, int successfulShards, int failedShards) { - this.index = index; - this.successfulShards = successfulShards; - this.failedShards = failedShards; - } - - IndexReplicationPingResponse() { - - } - - public String index() { - return index; - } - - public int successfulShards() { - return successfulShards; - } - - public int failedShards() { - return failedShards; - } - - public int totalShards() { - return successfulShards + failedShards; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - index = in.readUTF(); - successfulShards = in.readVInt(); - failedShards = in.readVInt(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeUTF(index); - out.writeVInt(successfulShards); - out.writeVInt(failedShards); - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ReplicationPingRequest.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ReplicationPingRequest.java deleted file mode 100644 index 0de2c47179d..00000000000 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ReplicationPingRequest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.ping.replication; - -import org.elasticsearch.action.support.replication.IndicesReplicationOperationRequest; -import org.elasticsearch.action.support.replication.ReplicationType; -import org.elasticsearch.common.unit.TimeValue; - -/** - * - */ -public class ReplicationPingRequest extends IndicesReplicationOperationRequest { - - public ReplicationPingRequest(String... indices) { - this.indices = indices; - } - - public ReplicationPingRequest() { - - } - - @Override - public ReplicationPingRequest listenerThreaded(boolean threadedListener) { - super.listenerThreaded(threadedListener); - return this; - } - - public ReplicationPingRequest replicationType(ReplicationType replicationType) { - this.replicationType = replicationType; - return this; - } - - public ReplicationPingRequest timeout(TimeValue timeout) { - this.timeout = timeout; - return this; - } - - public ReplicationPingRequest timeout(String timeout) { - return timeout(TimeValue.parseTimeValue(timeout, null)); - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ReplicationPingRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ReplicationPingRequestBuilder.java deleted file mode 100644 index c3151391caa..00000000000 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ReplicationPingRequestBuilder.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.ping.replication; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.support.BaseClusterRequestBuilder; -import org.elasticsearch.action.support.replication.ReplicationType; -import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.common.unit.TimeValue; - -/** - * - */ -public class ReplicationPingRequestBuilder extends BaseClusterRequestBuilder { - - public ReplicationPingRequestBuilder(ClusterAdminClient clusterClient) { - super(clusterClient, new ReplicationPingRequest()); - } - - public ReplicationPingRequestBuilder setIndices(String... indices) { - request.indices(indices); - return this; - } - - public ReplicationPingRequestBuilder setListenerThreaded(boolean threadedListener) { - request.listenerThreaded(threadedListener); - return this; - } - - public ReplicationPingRequestBuilder setReplicationType(ReplicationType replicationType) { - request.replicationType(replicationType); - return this; - } - - public ReplicationPingRequestBuilder setTimeout(TimeValue timeout) { - request.timeout(timeout); - return this; - } - - public ReplicationPingRequestBuilder setTimeout(String timeout) { - request.timeout(timeout); - return this; - } - - @Override - protected void doExecute(ActionListener listener) { - client.ping(request, listener); - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ReplicationPingResponse.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ReplicationPingResponse.java deleted file mode 100644 index 9535c9248fb..00000000000 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ReplicationPingResponse.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.ping.replication; - -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -/** - * - */ -public class ReplicationPingResponse implements ActionResponse, Streamable { - - private Map responses = new HashMap(); - - ReplicationPingResponse() { - - } - - public Map indices() { - return responses; - } - - public IndexReplicationPingResponse index(String index) { - return responses.get(index); - } - - @Override - public void readFrom(StreamInput in) throws IOException { - int size = in.readVInt(); - for (int i = 0; i < size; i++) { - IndexReplicationPingResponse response = new IndexReplicationPingResponse(); - response.readFrom(in); - responses.put(response.index(), response); - } - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(responses.size()); - for (IndexReplicationPingResponse response : responses.values()) { - response.writeTo(out); - } - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ShardReplicationPingRequest.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ShardReplicationPingRequest.java deleted file mode 100644 index 498b0f43d41..00000000000 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ShardReplicationPingRequest.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.ping.replication; - -import org.elasticsearch.action.support.replication.ReplicationType; -import org.elasticsearch.action.support.replication.ShardReplicationOperationRequest; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; - -import java.io.IOException; - -/** - * - */ -public class ShardReplicationPingRequest extends ShardReplicationOperationRequest { - - private int shardId; - - public ShardReplicationPingRequest(IndexReplicationPingRequest request, int shardId) { - this(request.index(), shardId); - timeout = request.timeout(); - replicationType(request.replicationType()); - } - - public ShardReplicationPingRequest(String index, int shardId) { - this.index = index; - this.shardId = shardId; - } - - ShardReplicationPingRequest() { - } - - public int shardId() { - return this.shardId; - } - - @Override - public ShardReplicationPingRequest listenerThreaded(boolean threadedListener) { - super.listenerThreaded(threadedListener); - return this; - } - - @Override - public ShardReplicationPingRequest operationThreaded(boolean threadedOperation) { - super.operationThreaded(threadedOperation); - return this; - } - - @Override - public ShardReplicationPingRequest replicationType(ReplicationType replicationType) { - super.replicationType(replicationType); - return this; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - shardId = in.readVInt(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeVInt(shardId); - } - - @Override - public String toString() { - return "replication_ping {[" + index + "][" + shardId + "]}"; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportIndexReplicationPingAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportIndexReplicationPingAction.java deleted file mode 100644 index 267bffe3271..00000000000 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportIndexReplicationPingAction.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.ping.replication; - -import org.elasticsearch.action.support.replication.TransportIndexReplicationOperationAction; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.routing.GroupShardsIterator; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.indices.IndexMissingException; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; - -import java.util.concurrent.atomic.AtomicReferenceArray; - -/** - */ -public class TransportIndexReplicationPingAction extends TransportIndexReplicationOperationAction { - - @Inject - public TransportIndexReplicationPingAction(Settings settings, ClusterService clusterService, - TransportService transportService, ThreadPool threadPool, - TransportShardReplicationPingAction shardReplicationPingAction) { - super(settings, transportService, clusterService, threadPool, shardReplicationPingAction); - } - - @Override - protected IndexReplicationPingRequest newRequestInstance() { - return new IndexReplicationPingRequest(); - } - - @Override - protected IndexReplicationPingResponse newResponseInstance(IndexReplicationPingRequest request, AtomicReferenceArray shardsResponses) { - int successfulShards = 0; - int failedShards = 0; - for (int i = 0; i < shardsResponses.length(); i++) { - if (shardsResponses.get(i) == null) { - failedShards++; - } else { - successfulShards++; - } - } - return new IndexReplicationPingResponse(request.index(), successfulShards, failedShards); - } - - @Override - protected boolean accumulateExceptions() { - return false; - } - - @Override - protected String transportAction() { - return "ping/replication/index"; - } - - @Override - protected GroupShardsIterator shards(IndexReplicationPingRequest indexRequest) { - IndexRoutingTable indexRouting = clusterService.state().routingTable().index(indexRequest.index()); - if (indexRouting == null) { - throw new IndexMissingException(new Index(indexRequest.index())); - } - return indexRouting.groupByShardsIt(); - } - - @Override - protected ShardReplicationPingRequest newShardRequestInstance(IndexReplicationPingRequest indexRequest, int shardId) { - return new ShardReplicationPingRequest(indexRequest, shardId); - } - - @Override - protected ClusterBlockException checkGlobalBlock(ClusterState state, IndexReplicationPingRequest request) { - return state.blocks().globalBlockedException(ClusterBlockLevel.READ); - } - - @Override - protected ClusterBlockException checkRequestBlock(ClusterState state, IndexReplicationPingRequest request) { - return state.blocks().indexBlockedException(ClusterBlockLevel.READ, request.index()); - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportReplicationPingAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportReplicationPingAction.java deleted file mode 100644 index 53a0e15c031..00000000000 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportReplicationPingAction.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.ping.replication; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.support.replication.TransportIndicesReplicationOperationAction; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; - -import java.util.Set; -import java.util.concurrent.atomic.AtomicReferenceArray; - -/** - */ -public class TransportReplicationPingAction extends TransportIndicesReplicationOperationAction { - - @Inject - public TransportReplicationPingAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, TransportIndexReplicationPingAction indexAction) { - super(settings, transportService, clusterService, threadPool, indexAction); - } - - @Override - protected ReplicationPingRequest newRequestInstance() { - return new ReplicationPingRequest(); - } - - @Override - protected ReplicationPingResponse newResponseInstance(ReplicationPingRequest request, AtomicReferenceArray indexResponses) { - ReplicationPingResponse response = new ReplicationPingResponse(); - for (int i = 0; i < indexResponses.length(); i++) { - IndexReplicationPingResponse indexResponse = (IndexReplicationPingResponse) indexResponses.get(i); - if (indexResponse != null) { - response.indices().put(indexResponse.index(), indexResponse); - } - } - return response; - } - - @Override - protected boolean accumulateExceptions() { - return false; - } - - @Override - protected String transportAction() { - return TransportActions.Admin.Cluster.Ping.REPLICATION; - } - - @Override - protected IndexReplicationPingRequest newIndexRequestInstance(ReplicationPingRequest request, String index, Set routing) { - return new IndexReplicationPingRequest(request, index); - } - - @Override - protected ClusterBlockException checkGlobalBlock(ClusterState state, ReplicationPingRequest replicationPingRequest) { - return state.blocks().globalBlockedException(ClusterBlockLevel.READ); - } - - @Override - protected ClusterBlockException checkRequestBlock(ClusterState state, ReplicationPingRequest replicationPingRequest, String[] concreteIndices) { - return state.blocks().indicesBlockedException(ClusterBlockLevel.WRITE, concreteIndices); - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportShardReplicationPingAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportShardReplicationPingAction.java deleted file mode 100644 index d6e53b0f432..00000000000 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/TransportShardReplicationPingAction.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.ping.replication; - -import org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.action.shard.ShardStateAction; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.routing.ShardIterator; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class TransportShardReplicationPingAction extends TransportShardReplicationOperationAction { - - @Inject - public TransportShardReplicationPingAction(Settings settings, TransportService transportService, - ClusterService clusterService, IndicesService indicesService, ThreadPool threadPool, - ShardStateAction shardStateAction) { - super(settings, transportService, clusterService, indicesService, threadPool, shardStateAction); - } - - @Override - protected String executor() { - return ThreadPool.Names.CACHED; - } - - @Override - protected boolean checkWriteConsistency() { - return true; - } - - @Override - protected ShardReplicationPingRequest newRequestInstance() { - return new ShardReplicationPingRequest(); - } - - @Override - protected ShardReplicationPingRequest newReplicaRequestInstance() { - return new ShardReplicationPingRequest(); - } - - @Override - protected ShardReplicationPingResponse newResponseInstance() { - return new ShardReplicationPingResponse(); - } - - @Override - protected String transportAction() { - return "ping/replication/shard"; - } - - @Override - protected PrimaryResponse shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) { - return new PrimaryResponse(shardRequest.request, new ShardReplicationPingResponse(), null); - } - - @Override - protected void shardOperationOnReplica(ReplicaOperationRequest shardRequest) { - } - - @Override - protected ClusterBlockException checkGlobalBlock(ClusterState state, ShardReplicationPingRequest request) { - return state.blocks().globalBlockedException(ClusterBlockLevel.READ); - } - - @Override - protected ClusterBlockException checkRequestBlock(ClusterState state, ShardReplicationPingRequest request) { - return state.blocks().indexBlockedException(ClusterBlockLevel.READ, request.index()); - } - - @Override - protected ShardIterator shards(ClusterState clusterState, ShardReplicationPingRequest request) { - return clusterService.state().routingTable().index(request.index()).shard(request.shardId()).shardsIt(); - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/SinglePingRequest.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/SinglePingRequest.java deleted file mode 100644 index fbf5675ad05..00000000000 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/SinglePingRequest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.ping.single; - -import org.elasticsearch.action.support.single.shard.SingleShardOperationRequest; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; - -import java.io.IOException; - -/** - * - */ -public class SinglePingRequest extends SingleShardOperationRequest { - - protected String type; - protected String id; - - public SinglePingRequest(String index) { - super(index); - } - - public SinglePingRequest(String index, String type, String id) { - super(index); - this.type = type; - this.id = id; - } - - public SinglePingRequest() { - } - - public SinglePingRequest index(String index) { - this.index = index; - return this; - } - - public SinglePingRequest type(String type) { - this.type = type; - return this; - } - - public SinglePingRequest id(String id) { - this.id = id; - return this; - } - - @Override - public SinglePingRequest listenerThreaded(boolean threadedListener) { - super.listenerThreaded(threadedListener); - return this; - } - - @Override - public SinglePingRequest operationThreaded(boolean threadedOperation) { - super.operationThreaded(threadedOperation); - return this; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - type = in.readUTF(); - id = in.readUTF(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeUTF(type); - out.writeUTF(id); - } -} diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/SinglePingRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/SinglePingRequestBuilder.java deleted file mode 100644 index ffd70b03e3c..00000000000 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/SinglePingRequestBuilder.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.ping.single; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.support.BaseClusterRequestBuilder; -import org.elasticsearch.client.ClusterAdminClient; - -/** - * - */ -public class SinglePingRequestBuilder extends BaseClusterRequestBuilder { - - public SinglePingRequestBuilder(ClusterAdminClient clusterClient) { - super(clusterClient, new SinglePingRequest()); - } - - public SinglePingRequestBuilder setIndex(String index) { - request.index(index); - return this; - } - - public SinglePingRequestBuilder setType(String type) { - request.type(type); - return this; - } - - public SinglePingRequestBuilder setId(String id) { - request.id(id); - return this; - } - - public SinglePingRequestBuilder setListenerThreaded(boolean threadedListener) { - request.listenerThreaded(threadedListener); - return this; - } - - public SinglePingRequestBuilder setOperationThreaded(boolean threadedOperation) { - request.operationThreaded(threadedOperation); - return this; - } - - @Override - protected void doExecute(ActionListener listener) { - client.ping(request, listener); - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/TransportSinglePingAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/TransportSinglePingAction.java deleted file mode 100644 index 69969593937..00000000000 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/single/TransportSinglePingAction.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.ping.single; - -import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.support.single.shard.TransportShardSingleOperationAction; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.routing.ShardIterator; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class TransportSinglePingAction extends TransportShardSingleOperationAction { - - @Inject - public TransportSinglePingAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService) { - super(settings, threadPool, clusterService, transportService); - } - - @Override - protected String executor() { - return ThreadPool.Names.CACHED; - } - - @Override - protected String transportAction() { - return TransportActions.Admin.Cluster.Ping.SINGLE; - } - - @Override - protected ClusterBlockException checkGlobalBlock(ClusterState state, SinglePingRequest request) { - return state.blocks().globalBlockedException(ClusterBlockLevel.READ); - } - - @Override - protected ClusterBlockException checkRequestBlock(ClusterState state, SinglePingRequest singlePingRequest) { - return null; - } - - @Override - protected ShardIterator shards(ClusterState state, SinglePingRequest request) throws ElasticSearchException { - return clusterService.operationRouting() - .getShards(clusterService.state(), request.index(), request.type, request.id, null, null); - } - - @Override - protected SinglePingResponse shardOperation(SinglePingRequest request, int shardId) throws ElasticSearchException { - return new SinglePingResponse(); - } - - @Override - protected SinglePingRequest newRequest() { - return new SinglePingRequest(); - } - - @Override - protected SinglePingResponse newResponse() { - return new SinglePingResponse(); - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteAction.java new file mode 100644 index 00000000000..9c23d1ccbc2 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.reroute; + +import org.elasticsearch.action.admin.cluster.ClusterAction; +import org.elasticsearch.client.ClusterAdminClient; + +/** + */ +public class ClusterRerouteAction extends ClusterAction { + + public static final ClusterRerouteAction INSTANCE = new ClusterRerouteAction(); + public static final String NAME = "cluster/reroute"; + + private ClusterRerouteAction() { + super(NAME); + } + + @Override + public ClusterRerouteResponse newResponse() { + return new ClusterRerouteResponse(); + } + + @Override + public ClusterRerouteRequestBuilder newRequestBuilder(ClusterAdminClient client) { + return new ClusterRerouteRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java index 61ecfd38129..aa75b700453 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.admin.cluster.reroute; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -57,7 +56,7 @@ public class TransportClusterRerouteAction extends TransportMasterNodeOperationA @Override protected String transportAction() { - return TransportActions.Admin.Cluster.REROUTE; + return ClusterRerouteAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsAction.java new file mode 100644 index 00000000000..906cc3495b5 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.settings; + +import org.elasticsearch.action.admin.cluster.ClusterAction; +import org.elasticsearch.client.ClusterAdminClient; + +/** + */ +public class ClusterUpdateSettingsAction extends ClusterAction { + + public static final ClusterUpdateSettingsAction INSTANCE = new ClusterUpdateSettingsAction(); + public static final String NAME = "cluster/settings/update"; + + private ClusterUpdateSettingsAction() { + super(NAME); + } + + @Override + public ClusterUpdateSettingsResponse newResponse() { + return new ClusterUpdateSettingsResponse(); + } + + @Override + public ClusterUpdateSettingsRequestBuilder newRequestBuilder(ClusterAdminClient client) { + return new ClusterUpdateSettingsRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java index 0d6d5e847fb..c00712cb57a 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.admin.cluster.settings; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -63,7 +62,7 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeOpe @Override protected String transportAction() { - return TransportActions.Admin.Cluster.UPDATE_SETTINGS; + return ClusterUpdateSettingsAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateAction.java new file mode 100644 index 00000000000..59facb5c250 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.state; + +import org.elasticsearch.action.admin.cluster.ClusterAction; +import org.elasticsearch.client.ClusterAdminClient; + +/** + */ +public class ClusterStateAction extends ClusterAction { + + public static final ClusterStateAction INSTANCE = new ClusterStateAction(); + public static final String NAME = "cluster/state"; + + private ClusterStateAction() { + super(NAME); + } + + @Override + public ClusterStateResponse newResponse() { + return new ClusterStateResponse(); + } + + @Override + public ClusterStateRequestBuilder newRequestBuilder(ClusterAdminClient client) { + return new ClusterStateRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java index dcdfdc519db..48f3f4d0de0 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.admin.cluster.state; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; @@ -57,7 +56,7 @@ public class TransportClusterStateAction extends TransportMasterNodeOperationAct @Override protected String transportAction() { - return TransportActions.Admin.Cluster.STATE; + return ClusterStateAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/IndicesAction.java b/src/main/java/org/elasticsearch/action/admin/indices/IndicesAction.java new file mode 100644 index 00000000000..6d215ef2369 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/IndicesAction.java @@ -0,0 +1,39 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.GenericAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + * Indices action (used with {@link IndicesAdminClient} API. + */ +public abstract class IndicesAction> + extends GenericAction { + + protected IndicesAction(String name) { + super(name); + } + + public abstract RequestBuilder newRequestBuilder(IndicesAdminClient client); +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesAction.java b/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesAction.java new file mode 100644 index 00000000000..6825cc7893b --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.alias; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class IndicesAliasesAction extends IndicesAction { + + public static final IndicesAliasesAction INSTANCE = new IndicesAliasesAction(); + public static final String NAME = "indices/aliases"; + + private IndicesAliasesAction() { + super(NAME); + } + + @Override + public IndicesAliasesResponse newResponse() { + return new IndicesAliasesResponse(); + } + + @Override + public IndicesAliasesRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new IndicesAliasesRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index 25cee1ae4d5..9b4d2b5486b 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -21,7 +21,6 @@ package org.elasticsearch.action.admin.indices.alias; import com.google.common.collect.Sets; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -59,7 +58,7 @@ public class TransportIndicesAliasesAction extends TransportMasterNodeOperationA @Override protected String transportAction() { - return TransportActions.Admin.Indices.ALIASES; + return IndicesAliasesAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java b/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java new file mode 100644 index 00000000000..9488a983fc9 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.analyze; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class AnalyzeAction extends IndicesAction { + + public static final AnalyzeAction INSTANCE = new AnalyzeAction(); + public static final String NAME = "indices/analyze"; + + private AnalyzeAction() { + super(NAME); + } + + @Override + public AnalyzeResponse newResponse() { + return new AnalyzeResponse(); + } + + @Override + public AnalyzeRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new AnalyzeRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java index 8fa274268f9..9160054f0b8 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java @@ -28,6 +28,10 @@ import org.elasticsearch.client.IndicesAdminClient; */ public class AnalyzeRequestBuilder extends BaseIndicesRequestBuilder { + public AnalyzeRequestBuilder(IndicesAdminClient indicesClient) { + super(indicesClient, new AnalyzeRequest()); + } + public AnalyzeRequestBuilder(IndicesAdminClient indicesClient, String index, String text) { super(indicesClient, new AnalyzeRequest(index, text)); } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index 1a7667419c5..24b696795d5 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -28,7 +28,6 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ElasticSearchIllegalArgumentException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.single.custom.TransportSingleCustomOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -85,7 +84,7 @@ public class TransportAnalyzeAction extends TransportSingleCustomOperationAction @Override protected String transportAction() { - return TransportActions.Admin.Indices.ANALYZE; + return AnalyzeAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheAction.java b/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheAction.java new file mode 100644 index 00000000000..6baff330749 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.cache.clear; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class ClearIndicesCacheAction extends IndicesAction { + + public static final ClearIndicesCacheAction INSTANCE = new ClearIndicesCacheAction(); + public static final String NAME = "indices/cache/clear"; + + private ClearIndicesCacheAction() { + super(NAME); + } + + @Override + public ClearIndicesCacheResponse newResponse() { + return new ClearIndicesCacheResponse(); + } + + @Override + public ClearIndicesCacheRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new ClearIndicesCacheRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java b/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java index 767b594a199..9421d51e1bb 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java @@ -21,7 +21,6 @@ package org.elasticsearch.action.admin.indices.cache.clear; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; @@ -64,7 +63,7 @@ public class TransportClearIndicesCacheAction extends TransportBroadcastOperatio @Override protected String transportAction() { - return TransportActions.Admin.Indices.Cache.CLEAR; + return ClearIndicesCacheAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexAction.java b/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexAction.java new file mode 100644 index 00000000000..c5518a8300f --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.close; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class CloseIndexAction extends IndicesAction { + + public static final CloseIndexAction INSTANCE = new CloseIndexAction(); + public static final String NAME = "indices/close"; + + private CloseIndexAction() { + super(NAME); + } + + @Override + public CloseIndexResponse newResponse() { + return new CloseIndexResponse(); + } + + @Override + public CloseIndexRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new CloseIndexRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java index 3726228ab77..36dd4d8de1b 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java @@ -66,6 +66,11 @@ public class CloseIndexRequest extends MasterNodeOperationRequest { return index; } + public CloseIndexRequest index(String index) { + this.index = index; + return this; + } + /** * Timeout to wait for the index deletion to be acknowledged by current cluster nodes. Defaults * to 10s. diff --git a/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestBuilder.java index 656c656b38e..d4b4f253f28 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestBuilder.java @@ -29,10 +29,19 @@ import org.elasticsearch.common.unit.TimeValue; */ public class CloseIndexRequestBuilder extends BaseIndicesRequestBuilder { + public CloseIndexRequestBuilder(IndicesAdminClient indicesClient) { + super(indicesClient, new CloseIndexRequest()); + } + public CloseIndexRequestBuilder(IndicesAdminClient indicesClient, String index) { super(indicesClient, new CloseIndexRequest(index)); } + public CloseIndexRequestBuilder setIndex(String index) { + request.index(index); + return this; + } + /** * Timeout to wait for the operation to be acknowledged by current cluster nodes. Defaults * to 10s. diff --git a/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java b/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java index 73c46bbae87..a326a775821 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.admin.indices.close; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -37,8 +36,6 @@ import java.util.concurrent.atomic.AtomicReference; /** * Delete index action. - * - * */ public class TransportCloseIndexAction extends TransportMasterNodeOperationAction { @@ -58,7 +55,7 @@ public class TransportCloseIndexAction extends TransportMasterNodeOperationActio @Override protected String transportAction() { - return TransportActions.Admin.Indices.CLOSE; + return CloseIndexAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexAction.java b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexAction.java new file mode 100644 index 00000000000..3fb3fe33216 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.create; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class CreateIndexAction extends IndicesAction { + + public static final CreateIndexAction INSTANCE = new CreateIndexAction(); + public static final String NAME = "indices/create"; + + private CreateIndexAction() { + super(NAME); + } + + @Override + public CreateIndexResponse newResponse() { + return new CreateIndexResponse(); + } + + @Override + public CreateIndexRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new CreateIndexRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 1ba3cebd143..eb88f013532 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -99,6 +99,11 @@ public class CreateIndexRequest extends MasterNodeOperationRequest { return index; } + public CreateIndexRequest index(String index) { + this.index = index; + return this; + } + /** * The settings to created the index with. */ diff --git a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java index 0af6832f6e7..ccf8652fcac 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java @@ -33,10 +33,19 @@ import java.util.Map; */ public class CreateIndexRequestBuilder extends BaseIndicesRequestBuilder { + public CreateIndexRequestBuilder(IndicesAdminClient indicesClient) { + super(indicesClient, new CreateIndexRequest()); + } + public CreateIndexRequestBuilder(IndicesAdminClient indicesClient, String index) { super(indicesClient, new CreateIndexRequest(index)); } + public CreateIndexRequestBuilder setIndex(String index) { + request.index(index); + return this; + } + /** * The settings to created the index with. */ diff --git a/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java index b2367985261..3e8ecdab8ed 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.admin.indices.create; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -37,8 +36,6 @@ import java.util.concurrent.atomic.AtomicReference; /** * Create index action. - * - * */ public class TransportCreateIndexAction extends TransportMasterNodeOperationAction { @@ -58,7 +55,7 @@ public class TransportCreateIndexAction extends TransportMasterNodeOperationActi @Override protected String transportAction() { - return TransportActions.Admin.Indices.CREATE; + return CreateIndexAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexAction.java b/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexAction.java new file mode 100644 index 00000000000..c640b4783b2 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.delete; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class DeleteIndexAction extends IndicesAction { + + public static final DeleteIndexAction INSTANCE = new DeleteIndexAction(); + public static final String NAME = "indices/delete"; + + private DeleteIndexAction() { + super(NAME); + } + + @Override + public DeleteIndexResponse newResponse() { + return new DeleteIndexResponse(); + } + + @Override + public DeleteIndexRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new DeleteIndexRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java index 7a41f517d23..ffb60bc280e 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java @@ -22,7 +22,6 @@ package org.elasticsearch.action.admin.indices.delete; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequest; import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingResponse; import org.elasticsearch.action.admin.indices.mapping.delete.TransportDeleteMappingAction; @@ -44,8 +43,6 @@ import java.util.concurrent.atomic.AtomicReference; /** * Delete index action. - * - * */ public class TransportDeleteIndexAction extends TransportMasterNodeOperationAction { @@ -72,7 +69,7 @@ public class TransportDeleteIndexAction extends TransportMasterNodeOperationActi @Override protected String transportAction() { - return TransportActions.Admin.Indices.DELETE; + return DeleteIndexAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsAction.java b/src/main/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsAction.java new file mode 100644 index 00000000000..d9d20da12e0 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.exists; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class IndicesExistsAction extends IndicesAction { + + public static final IndicesExistsAction INSTANCE = new IndicesExistsAction(); + public static final String NAME = "indices/exists"; + + private IndicesExistsAction() { + super(NAME); + } + + @Override + public IndicesExistsResponse newResponse() { + return new IndicesExistsResponse(); + } + + @Override + public IndicesExistsRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new IndicesExistsRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/exists/TransportIndicesExistsAction.java b/src/main/java/org/elasticsearch/action/admin/indices/exists/TransportIndicesExistsAction.java index 30b68140eb6..2c8b38513a2 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/exists/TransportIndicesExistsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/exists/TransportIndicesExistsAction.java @@ -21,7 +21,6 @@ package org.elasticsearch.action.admin.indices.exists; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -34,8 +33,6 @@ import org.elasticsearch.transport.TransportService; /** * Indices exists action. - * - * */ public class TransportIndicesExistsAction extends TransportMasterNodeOperationAction { @@ -52,7 +49,7 @@ public class TransportIndicesExistsAction extends TransportMasterNodeOperationAc @Override protected String transportAction() { - return TransportActions.Admin.Indices.EXISTS; + return IndicesExistsAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushAction.java b/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushAction.java new file mode 100644 index 00000000000..9279d1b87ea --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.flush; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class FlushAction extends IndicesAction { + + public static final FlushAction INSTANCE = new FlushAction(); + public static final String NAME = "indices/flush"; + + private FlushAction() { + super(NAME); + } + + @Override + public FlushResponse newResponse() { + return new FlushResponse(); + } + + @Override + public FlushRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new FlushRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java b/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java index 05965014b3f..4025967da55 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java @@ -21,7 +21,6 @@ package org.elasticsearch.action.admin.indices.flush; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; @@ -64,7 +63,7 @@ public class TransportFlushAction extends TransportBroadcastOperationAction { + + public static final GatewaySnapshotAction INSTANCE = new GatewaySnapshotAction(); + public static final String NAME = "indices/gateway/snapshot"; + + private GatewaySnapshotAction() { + super(NAME); + } + + @Override + public GatewaySnapshotResponse newResponse() { + return new GatewaySnapshotResponse(); + } + + @Override + public GatewaySnapshotRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new GatewaySnapshotRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/gateway/snapshot/TransportGatewaySnapshotAction.java b/src/main/java/org/elasticsearch/action/admin/indices/gateway/snapshot/TransportGatewaySnapshotAction.java index d3374f31760..3b551e2495a 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/gateway/snapshot/TransportGatewaySnapshotAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/gateway/snapshot/TransportGatewaySnapshotAction.java @@ -22,7 +22,6 @@ package org.elasticsearch.action.admin.indices.gateway.snapshot; import com.google.common.collect.Lists; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; @@ -63,7 +62,7 @@ public class TransportGatewaySnapshotAction extends TransportBroadcastOperationA @Override protected String transportAction() { - return TransportActions.Admin.Indices.Gateway.SNAPSHOT; + return GatewaySnapshotAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/DeleteMappingAction.java b/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/DeleteMappingAction.java new file mode 100644 index 00000000000..1ca3fda7fc8 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/DeleteMappingAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.mapping.delete; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class DeleteMappingAction extends IndicesAction { + + public static final DeleteMappingAction INSTANCE = new DeleteMappingAction(); + public static final String NAME = "indices/mapping/delete"; + + private DeleteMappingAction() { + super(NAME); + } + + @Override + public DeleteMappingResponse newResponse() { + return new DeleteMappingResponse(); + } + + @Override + public DeleteMappingRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new DeleteMappingRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/TransportDeleteMappingAction.java b/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/TransportDeleteMappingAction.java index 09639790c27..ec3b305c9f8 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/TransportDeleteMappingAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/TransportDeleteMappingAction.java @@ -21,7 +21,6 @@ package org.elasticsearch.action.admin.indices.mapping.delete; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.admin.indices.refresh.TransportRefreshAction; import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse; @@ -45,8 +44,6 @@ import java.util.concurrent.atomic.AtomicReference; /** * Delete mapping action. - * - * */ public class TransportDeleteMappingAction extends TransportMasterNodeOperationAction { @@ -73,7 +70,7 @@ public class TransportDeleteMappingAction extends TransportMasterNodeOperationAc @Override protected String transportAction() { - return TransportActions.Admin.Indices.Mapping.DELETE; + return DeleteMappingAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingAction.java b/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingAction.java new file mode 100644 index 00000000000..6fed38a3ab8 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.mapping.put; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class PutMappingAction extends IndicesAction { + + public static final PutMappingAction INSTANCE = new PutMappingAction(); + public static final String NAME = "indices/mapping/put"; + + private PutMappingAction() { + super(NAME); + } + + @Override + public PutMappingResponse newResponse() { + return new PutMappingResponse(); + } + + @Override + public PutMappingRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new PutMappingRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index 666465dd723..37be35991dd 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -21,7 +21,6 @@ package org.elasticsearch.action.admin.indices.mapping.put; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -38,8 +37,6 @@ import java.util.concurrent.atomic.AtomicReference; /** * Put mapping action. - * - * */ public class TransportPutMappingAction extends TransportMasterNodeOperationAction { @@ -59,7 +56,7 @@ public class TransportPutMappingAction extends TransportMasterNodeOperationActio @Override protected String transportAction() { - return TransportActions.Admin.Indices.Mapping.PUT; + return PutMappingAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexAction.java b/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexAction.java new file mode 100644 index 00000000000..8f15719f98b --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.open; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class OpenIndexAction extends IndicesAction { + + public static final OpenIndexAction INSTANCE = new OpenIndexAction(); + public static final String NAME = "indices/open"; + + private OpenIndexAction() { + super(NAME); + } + + @Override + public OpenIndexResponse newResponse() { + return new OpenIndexResponse(); + } + + @Override + public OpenIndexRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new OpenIndexRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequest.java index c64bd074c90..fe148615129 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequest.java @@ -66,6 +66,11 @@ public class OpenIndexRequest extends MasterNodeOperationRequest { return index; } + public OpenIndexRequest index(String index) { + this.index = index; + return this; + } + /** * Timeout to wait for the index deletion to be acknowledged by current cluster nodes. Defaults * to 10s. diff --git a/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequestBuilder.java index 64f425dfeb8..e87e4c8b652 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequestBuilder.java @@ -29,10 +29,19 @@ import org.elasticsearch.common.unit.TimeValue; */ public class OpenIndexRequestBuilder extends BaseIndicesRequestBuilder { + public OpenIndexRequestBuilder(IndicesAdminClient indicesClient) { + super(indicesClient, new OpenIndexRequest()); + } + public OpenIndexRequestBuilder(IndicesAdminClient indicesClient, String index) { super(indicesClient, new OpenIndexRequest(index)); } + public OpenIndexRequestBuilder setIndex(String index) { + request.index(index); + return this; + } + /** * Timeout to wait for the operation to be acknowledged by current cluster nodes. Defaults * to 10s. diff --git a/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java b/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java index 5dd400ce6c6..3a041389bf7 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.admin.indices.open; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -37,8 +36,6 @@ import java.util.concurrent.atomic.AtomicReference; /** * Delete index action. - * - * */ public class TransportOpenIndexAction extends TransportMasterNodeOperationAction { @@ -58,7 +55,7 @@ public class TransportOpenIndexAction extends TransportMasterNodeOperationAction @Override protected String transportAction() { - return TransportActions.Admin.Indices.OPEN; + return OpenIndexAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeAction.java b/src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeAction.java new file mode 100644 index 00000000000..a7ecb337296 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.optimize; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class OptimizeAction extends IndicesAction { + + public static final OptimizeAction INSTANCE = new OptimizeAction(); + public static final String NAME = "indices/optimize"; + + private OptimizeAction() { + super(NAME); + } + + @Override + public OptimizeResponse newResponse() { + return new OptimizeResponse(); + } + + @Override + public OptimizeRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new OptimizeRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/optimize/TransportOptimizeAction.java b/src/main/java/org/elasticsearch/action/admin/indices/optimize/TransportOptimizeAction.java index 71eaccfdf3d..3c479d890f3 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/optimize/TransportOptimizeAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/optimize/TransportOptimizeAction.java @@ -21,7 +21,6 @@ package org.elasticsearch.action.admin.indices.optimize; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; @@ -67,7 +66,7 @@ public class TransportOptimizeAction extends TransportBroadcastOperationAction { + + public static final RefreshAction INSTANCE = new RefreshAction(); + public static final String NAME = "indices/refresh"; + + private RefreshAction() { + super(NAME); + } + + @Override + public RefreshResponse newResponse() { + return new RefreshResponse(); + } + + @Override + public RefreshRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new RefreshRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java b/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java index 43beefdb099..7f64aa4d6f4 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java @@ -22,7 +22,6 @@ package org.elasticsearch.action.admin.indices.refresh; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; @@ -69,7 +68,7 @@ public class TransportRefreshAction extends TransportBroadcastOperationAction { + + public static final IndicesSegmentsAction INSTANCE = new IndicesSegmentsAction(); + public static final String NAME = "indices/segments"; + + private IndicesSegmentsAction() { + super(NAME); + } + + @Override + public IndicesSegmentResponse newResponse() { + return new IndicesSegmentResponse(); + } + + @Override + public IndicesSegmentsRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new IndicesSegmentsRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java b/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java index bdb295f935f..bc04afd6bbe 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java @@ -21,7 +21,6 @@ package org.elasticsearch.action.admin.indices.segments; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationRequest; @@ -69,7 +68,7 @@ public class TransportIndicesSegmentsAction extends TransportBroadcastOperationA @Override protected String transportAction() { - return TransportActions.Admin.Indices.SEGMENTS; + return IndicesSegmentsAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/settings/TransportUpdateSettingsAction.java b/src/main/java/org/elasticsearch/action/admin/indices/settings/TransportUpdateSettingsAction.java index 99b7de6cba2..3ec03ce6b2a 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/settings/TransportUpdateSettingsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/settings/TransportUpdateSettingsAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.admin.indices.settings; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -54,7 +53,7 @@ public class TransportUpdateSettingsAction extends TransportMasterNodeOperationA @Override protected String transportAction() { - return TransportActions.Admin.Indices.UPDATE_SETTINGS; + return UpdateSettingsAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/settings/UpdateSettingsAction.java b/src/main/java/org/elasticsearch/action/admin/indices/settings/UpdateSettingsAction.java new file mode 100644 index 00000000000..4654d49230c --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/settings/UpdateSettingsAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.settings; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class UpdateSettingsAction extends IndicesAction { + + public static final UpdateSettingsAction INSTANCE = new UpdateSettingsAction(); + public static final String NAME = "indices/settings/update"; + + private UpdateSettingsAction() { + super(NAME); + } + + @Override + public UpdateSettingsResponse newResponse() { + return new UpdateSettingsResponse(); + } + + @Override + public UpdateSettingsRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new UpdateSettingsRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsAction.java b/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsAction.java new file mode 100644 index 00000000000..e20ece12f0f --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.stats; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class IndicesStatsAction extends IndicesAction { + + public static final IndicesStatsAction INSTANCE = new IndicesStatsAction(); + public static final String NAME = "indices/stats"; + + private IndicesStatsAction() { + super(NAME); + } + + @Override + public IndicesStats newResponse() { + return new IndicesStats(); + } + + @Override + public IndicesStatsRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new IndicesStatsRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java index 251420d67b2..4a6794004b3 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java @@ -22,7 +22,6 @@ package org.elasticsearch.action.admin.indices.stats; import com.google.common.collect.Lists; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationRequest; @@ -69,7 +68,7 @@ public class TransportIndicesStatsAction extends TransportBroadcastOperationActi @Override protected String transportAction() { - return TransportActions.Admin.Indices.STATS; + return IndicesStatsAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/status/IndicesStatusAction.java b/src/main/java/org/elasticsearch/action/admin/indices/status/IndicesStatusAction.java new file mode 100644 index 00000000000..9b5369db1b9 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/status/IndicesStatusAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.status; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class IndicesStatusAction extends IndicesAction { + + public static final IndicesStatusAction INSTANCE = new IndicesStatusAction(); + public static final String NAME = "indices/status"; + + private IndicesStatusAction() { + super(NAME); + } + + @Override + public IndicesStatusResponse newResponse() { + return new IndicesStatusResponse(); + } + + @Override + public IndicesStatusRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new IndicesStatusRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/status/TransportIndicesStatusAction.java b/src/main/java/org/elasticsearch/action/admin/indices/status/TransportIndicesStatusAction.java index 59865266618..c2378c1fc20 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/status/TransportIndicesStatusAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/status/TransportIndicesStatusAction.java @@ -21,7 +21,6 @@ package org.elasticsearch.action.admin.indices.status; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationRequest; @@ -78,7 +77,7 @@ public class TransportIndicesStatusAction extends TransportBroadcastOperationAct @Override protected String transportAction() { - return TransportActions.Admin.Indices.STATUS; + return IndicesStatusAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateAction.java b/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateAction.java new file mode 100644 index 00000000000..d528ecea7f1 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.template.delete; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class DeleteIndexTemplateAction extends IndicesAction { + + public static final DeleteIndexTemplateAction INSTANCE = new DeleteIndexTemplateAction(); + public static final String NAME = "indices/template/delete"; + + private DeleteIndexTemplateAction() { + super(NAME); + } + + @Override + public DeleteIndexTemplateResponse newResponse() { + return new DeleteIndexTemplateResponse(); + } + + @Override + public DeleteIndexTemplateRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new DeleteIndexTemplateRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java index d4c8c14d1a0..cd5915da31e 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java @@ -29,6 +29,10 @@ import org.elasticsearch.common.unit.TimeValue; */ public class DeleteIndexTemplateRequestBuilder extends BaseIndicesRequestBuilder { + public DeleteIndexTemplateRequestBuilder(IndicesAdminClient indicesClient) { + super(indicesClient, new DeleteIndexTemplateRequest()); + } + public DeleteIndexTemplateRequestBuilder(IndicesAdminClient indicesClient, String name) { super(indicesClient, new DeleteIndexTemplateRequest(name)); } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java b/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java index 2e9d8031450..77e2a69a886 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.admin.indices.template.delete; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -56,7 +55,7 @@ public class TransportDeleteIndexTemplateAction extends TransportMasterNodeOpera @Override protected String transportAction() { - return TransportActions.Admin.Indices.Template.DELETE; + return DeleteIndexTemplateAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateAction.java b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateAction.java new file mode 100644 index 00000000000..7536f2ae9ee --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.template.put; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class PutIndexTemplateAction extends IndicesAction { + + public static final PutIndexTemplateAction INSTANCE = new PutIndexTemplateAction(); + public static final String NAME = "indices/template/put"; + + private PutIndexTemplateAction() { + super(NAME); + } + + @Override + public PutIndexTemplateResponse newResponse() { + return new PutIndexTemplateResponse(); + } + + @Override + public PutIndexTemplateRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new PutIndexTemplateRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java index 6c1c7a737cd..d1e3f181bfe 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java @@ -33,6 +33,10 @@ import java.util.Map; */ public class PutIndexTemplateRequestBuilder extends BaseIndicesRequestBuilder { + public PutIndexTemplateRequestBuilder(IndicesAdminClient indicesClient) { + super(indicesClient, new PutIndexTemplateRequest()); + } + public PutIndexTemplateRequestBuilder(IndicesAdminClient indicesClient, String name) { super(indicesClient, new PutIndexTemplateRequest(name)); } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java index 234618901f6..987df086f56 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.admin.indices.template.put; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -56,7 +55,7 @@ public class TransportPutIndexTemplateAction extends TransportMasterNodeOperatio @Override protected String transportAction() { - return TransportActions.Admin.Indices.Template.PUT; + return PutIndexTemplateAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java index 89e86ea901e..47081e5eb09 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java @@ -22,7 +22,6 @@ package org.elasticsearch.action.admin.indices.validate.query; import jsr166y.ThreadLocalRandom; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; @@ -67,7 +66,7 @@ public class TransportValidateQueryAction extends TransportBroadcastOperationAct @Override protected String transportAction() { - return TransportActions.Admin.Indices.Validate.QUERY; + return ValidateQueryAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryAction.java b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryAction.java new file mode 100644 index 00000000000..899a5e0374a --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.validate.query; + +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.client.IndicesAdminClient; + +/** + */ +public class ValidateQueryAction extends IndicesAction { + + public static final ValidateQueryAction INSTANCE = new ValidateQueryAction(); + public static final String NAME = "indices/validate/query"; + + private ValidateQueryAction() { + super(NAME); + } + + @Override + public ValidateQueryResponse newResponse() { + return new ValidateQueryResponse(); + } + + @Override + public ValidateQueryRequestBuilder newRequestBuilder(IndicesAdminClient client) { + return new ValidateQueryRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/client/transport/action/bulk/ClientTransportBulkAction.java b/src/main/java/org/elasticsearch/action/bulk/BulkAction.java similarity index 54% rename from src/main/java/org/elasticsearch/client/transport/action/bulk/ClientTransportBulkAction.java rename to src/main/java/org/elasticsearch/action/bulk/BulkAction.java index e72a0fc3a7d..b6bc8c2f072 100644 --- a/src/main/java/org/elasticsearch/client/transport/action/bulk/ClientTransportBulkAction.java +++ b/src/main/java/org/elasticsearch/action/bulk/BulkAction.java @@ -17,34 +17,35 @@ * under the License. */ -package org.elasticsearch.client.transport.action.bulk; +package org.elasticsearch.action.bulk; -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.action.Action; +import org.elasticsearch.client.Client; import org.elasticsearch.transport.TransportRequestOptions; -import org.elasticsearch.transport.TransportService; /** - * */ -public class ClientTransportBulkAction extends BaseClientTransportAction { +public class BulkAction extends Action { - @Inject - public ClientTransportBulkAction(Settings settings, TransportService transportService) { - super(settings, transportService, BulkResponse.class); + public static final BulkAction INSTANCE = new BulkAction(); + public static final String NAME = "bulk"; + + private BulkAction() { + super(NAME); } @Override - protected String action() { - return TransportActions.BULK; + public BulkResponse newResponse() { + return new BulkResponse(); } @Override - protected TransportRequestOptions options() { + public BulkRequestBuilder newRequestBuilder(Client client) { + return new BulkRequestBuilder(client); + } + + @Override + public TransportRequestOptions options() { return TransportRequestOptions.options().withLowType().withCompress(true); } } diff --git a/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 0331b98fa2a..c95f9506847 100644 --- a/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -25,7 +25,6 @@ import com.google.common.collect.Sets; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; @@ -80,7 +79,7 @@ public class TransportBulkAction extends TransportAction { - BroadcastShardPingRequest() { + public static final CountAction INSTANCE = new CountAction(); + public static final String NAME = "count"; - } - - public BroadcastShardPingRequest(String index, int shardId) { - super(index, shardId); + private CountAction() { + super(NAME); } @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); + public CountResponse newResponse() { + return new CountResponse(); } @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); + public CountRequestBuilder newRequestBuilder(Client client) { + return new CountRequestBuilder(client); } -} \ No newline at end of file +} diff --git a/src/main/java/org/elasticsearch/action/count/TransportCountAction.java b/src/main/java/org/elasticsearch/action/count/TransportCountAction.java index 503351d4597..9dca91191ed 100644 --- a/src/main/java/org/elasticsearch/action/count/TransportCountAction.java +++ b/src/main/java/org/elasticsearch/action/count/TransportCountAction.java @@ -21,7 +21,6 @@ package org.elasticsearch.action.count; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; @@ -65,7 +64,7 @@ public class TransportCountAction extends TransportBroadcastOperationAction { - BroadcastShardPingResponse() { + public static final DeleteAction INSTANCE = new DeleteAction(); + public static final String NAME = "delete"; - } - - public BroadcastShardPingResponse(String index, int shardId) { - super(index, shardId); + private DeleteAction() { + super(NAME); } @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); + public DeleteResponse newResponse() { + return new DeleteResponse(); } @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); + public DeleteRequestBuilder newRequestBuilder(Client client) { + return new DeleteRequestBuilder(client); } -} \ No newline at end of file +} diff --git a/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java b/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java index c2bffc94d62..dbbbcb0e288 100644 --- a/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java @@ -32,6 +32,10 @@ import org.elasticsearch.index.VersionType; */ public class DeleteRequestBuilder extends BaseRequestBuilder { + public DeleteRequestBuilder(Client client) { + super(client, new DeleteRequest()); + } + public DeleteRequestBuilder(Client client, @Nullable String index) { super(client, new DeleteRequest(index)); } diff --git a/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java b/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java index 4870e164297..22854eae3f8 100644 --- a/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java +++ b/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java @@ -21,7 +21,6 @@ package org.elasticsearch.action.delete; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; @@ -160,7 +159,7 @@ public class TransportDeleteAction extends TransportShardReplicationOperationAct @Override protected String transportAction() { - return TransportActions.DELETE; + return DeleteAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryAction.java b/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryAction.java new file mode 100644 index 00000000000..af6b788985f --- /dev/null +++ b/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.deletebyquery; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.Client; + +/** + */ +public class DeleteByQueryAction extends Action { + + public static final DeleteByQueryAction INSTANCE = new DeleteByQueryAction(); + public static final String NAME = "deleteByQuery"; + + private DeleteByQueryAction() { + super(NAME); + } + + @Override + public DeleteByQueryResponse newResponse() { + return new DeleteByQueryResponse(); + } + + @Override + public DeleteByQueryRequestBuilder newRequestBuilder(Client client) { + return new DeleteByQueryRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java b/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java index 587b9548c25..bec8f09d365 100644 --- a/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java +++ b/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.deletebyquery; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.replication.TransportIndicesReplicationOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -67,7 +66,7 @@ public class TransportDeleteByQueryAction extends TransportIndicesReplicationOpe @Override protected String transportAction() { - return TransportActions.DELETE_BY_QUERY; + return DeleteByQueryAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/deletebyquery/TransportIndexDeleteByQueryAction.java b/src/main/java/org/elasticsearch/action/deletebyquery/TransportIndexDeleteByQueryAction.java index b815b38ee6d..c4a3a933789 100644 --- a/src/main/java/org/elasticsearch/action/deletebyquery/TransportIndexDeleteByQueryAction.java +++ b/src/main/java/org/elasticsearch/action/deletebyquery/TransportIndexDeleteByQueryAction.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.deletebyquery; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.replication.TransportIndexReplicationOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -70,7 +69,7 @@ public class TransportIndexDeleteByQueryAction extends TransportIndexReplication @Override protected String transportAction() { - return TransportActions.DELETE_BY_QUERY + "/index"; + return DeleteByQueryAction.NAME + "/index"; } @Override diff --git a/src/main/java/org/elasticsearch/action/deletebyquery/TransportShardDeleteByQueryAction.java b/src/main/java/org/elasticsearch/action/deletebyquery/TransportShardDeleteByQueryAction.java index 40ae24e4c01..7f2cd83497c 100644 --- a/src/main/java/org/elasticsearch/action/deletebyquery/TransportShardDeleteByQueryAction.java +++ b/src/main/java/org/elasticsearch/action/deletebyquery/TransportShardDeleteByQueryAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.deletebyquery; import org.elasticsearch.ElasticSearchIllegalStateException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -76,7 +75,7 @@ public class TransportShardDeleteByQueryAction extends TransportShardReplication @Override protected String transportAction() { - return TransportActions.DELETE_BY_QUERY + "/shard"; + return DeleteByQueryAction.NAME + "/shard"; } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ShardReplicationPingResponse.java b/src/main/java/org/elasticsearch/action/get/GetAction.java similarity index 60% rename from src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ShardReplicationPingResponse.java rename to src/main/java/org/elasticsearch/action/get/GetAction.java index 048f42e50fc..c9f39e2de8b 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/ping/replication/ShardReplicationPingResponse.java +++ b/src/main/java/org/elasticsearch/action/get/GetAction.java @@ -17,29 +17,29 @@ * under the License. */ -package org.elasticsearch.action.admin.cluster.ping.replication; +package org.elasticsearch.action.get; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; - -import java.io.IOException; +import org.elasticsearch.action.Action; +import org.elasticsearch.client.Client; /** - * */ -public class ShardReplicationPingResponse implements ActionResponse, Streamable { +public class GetAction extends Action { - ShardReplicationPingResponse() { + public static final GetAction INSTANCE = new GetAction(); + public static final String NAME = "get"; + private GetAction() { + super(NAME); } @Override - public void readFrom(StreamInput in) throws IOException { + public GetResponse newResponse() { + return new GetResponse(); } @Override - public void writeTo(StreamOutput out) throws IOException { + public GetRequestBuilder newRequestBuilder(Client client) { + return new GetRequestBuilder(client); } -} \ No newline at end of file +} diff --git a/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java b/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java index e61c3e344b0..a8152a064f4 100644 --- a/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java @@ -29,6 +29,10 @@ import org.elasticsearch.common.Nullable; */ public class GetRequestBuilder extends BaseRequestBuilder { + public GetRequestBuilder(Client client) { + super(client, new GetRequest()); + } + public GetRequestBuilder(Client client, @Nullable String index) { super(client, new GetRequest(index)); } diff --git a/src/main/java/org/elasticsearch/action/get/MultiGetAction.java b/src/main/java/org/elasticsearch/action/get/MultiGetAction.java new file mode 100644 index 00000000000..99e655cf2bc --- /dev/null +++ b/src/main/java/org/elasticsearch/action/get/MultiGetAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.get; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.Client; + +/** + */ +public class MultiGetAction extends Action { + + public static final MultiGetAction INSTANCE = new MultiGetAction(); + public static final String NAME = "mget"; + + private MultiGetAction() { + super(NAME); + } + + @Override + public MultiGetResponse newResponse() { + return new MultiGetResponse(); + } + + @Override + public MultiGetRequestBuilder newRequestBuilder(Client client) { + return new MultiGetRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/src/main/java/org/elasticsearch/action/get/TransportGetAction.java index 87e4ed0710c..3d337f9fc61 100644 --- a/src/main/java/org/elasticsearch/action/get/TransportGetAction.java +++ b/src/main/java/org/elasticsearch/action/get/TransportGetAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.get; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.single.shard.TransportShardSingleOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -62,7 +61,7 @@ public class TransportGetAction extends TransportShardSingleOperationAction { + + public static final IndexAction INSTANCE = new IndexAction(); + public static final String NAME = "index"; + + private IndexAction() { + super(NAME); + } + + @Override + public IndexResponse newResponse() { + return new IndexResponse(); + } + + @Override + public IndexRequestBuilder newRequestBuilder(Client client) { + return new IndexRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index 94837d4d5e2..6726e0dd13e 100644 --- a/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -37,6 +37,10 @@ import java.util.Map; */ public class IndexRequestBuilder extends BaseRequestBuilder { + public IndexRequestBuilder(Client client) { + super(client, new IndexRequest()); + } + public IndexRequestBuilder(Client client, @Nullable String index) { super(client, new IndexRequest(index)); } diff --git a/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java b/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java index 7eab00a4fb3..416b569dc0b 100644 --- a/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java +++ b/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java @@ -22,7 +22,6 @@ package org.elasticsearch.action.index; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.RoutingMissingException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; @@ -158,7 +157,7 @@ public class TransportIndexAction extends TransportShardReplicationOperationActi @Override protected String transportAction() { - return TransportActions.INDEX; + return IndexAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/mlt/MoreLikeThisAction.java b/src/main/java/org/elasticsearch/action/mlt/MoreLikeThisAction.java new file mode 100644 index 00000000000..b56758a0dab --- /dev/null +++ b/src/main/java/org/elasticsearch/action/mlt/MoreLikeThisAction.java @@ -0,0 +1,46 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.mlt; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; + +/** + */ +public class MoreLikeThisAction extends Action { + + public static final MoreLikeThisAction INSTANCE = new MoreLikeThisAction(); + public static final String NAME = "mlt"; + + private MoreLikeThisAction() { + super(NAME); + } + + @Override + public SearchResponse newResponse() { + return new SearchResponse(); + } + + @Override + public MoreLikeThisRequestBuilder newRequestBuilder(Client client) { + return new MoreLikeThisRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/mlt/MoreLikeThisRequestBuilder.java b/src/main/java/org/elasticsearch/action/mlt/MoreLikeThisRequestBuilder.java index 3e34539b3f5..9c9ab4d91e3 100644 --- a/src/main/java/org/elasticsearch/action/mlt/MoreLikeThisRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/mlt/MoreLikeThisRequestBuilder.java @@ -35,6 +35,10 @@ import java.util.Map; */ public class MoreLikeThisRequestBuilder extends BaseRequestBuilder { + public MoreLikeThisRequestBuilder(Client client) { + super(client, new MoreLikeThisRequest()); + } + public MoreLikeThisRequestBuilder(Client client, String index, String type, String id) { super(client, new MoreLikeThisRequest(index).type(type).id(id)); } diff --git a/src/main/java/org/elasticsearch/action/mlt/TransportMoreLikeThisAction.java b/src/main/java/org/elasticsearch/action/mlt/TransportMoreLikeThisAction.java index 68d51e964b9..3ddf3a5d196 100644 --- a/src/main/java/org/elasticsearch/action/mlt/TransportMoreLikeThisAction.java +++ b/src/main/java/org/elasticsearch/action/mlt/TransportMoreLikeThisAction.java @@ -23,7 +23,6 @@ import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.Term; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.get.TransportGetAction; @@ -78,7 +77,7 @@ public class TransportMoreLikeThisAction extends TransportAction { + + public static final PercolateAction INSTANCE = new PercolateAction(); + public static final String NAME = "percolate"; + + private PercolateAction() { + super(NAME); + } + + @Override + public PercolateResponse newResponse() { + return new PercolateResponse(); + } + + @Override + public PercolateRequestBuilder newRequestBuilder(Client client) { + return new PercolateRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java b/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java index 5d17c4b9437..5577516ead6 100644 --- a/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java @@ -32,6 +32,10 @@ import java.util.Map; */ public class PercolateRequestBuilder extends BaseRequestBuilder { + public PercolateRequestBuilder(Client client) { + super(client, new PercolateRequest()); + } + public PercolateRequestBuilder(Client client, String index, String type) { super(client, new PercolateRequest(index, type)); } diff --git a/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java b/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java index 560179eb7b8..383a287b98f 100644 --- a/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java +++ b/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.percolate; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.support.single.custom.TransportSingleCustomOperationAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -67,7 +66,7 @@ public class TransportPercolateAction extends TransportSingleCustomOperationActi @Override protected String transportAction() { - return TransportActions.PERCOLATE; + return PercolateAction.NAME; } @Override diff --git a/src/main/java/org/elasticsearch/action/search/SearchAction.java b/src/main/java/org/elasticsearch/action/search/SearchAction.java new file mode 100644 index 00000000000..fa5a1049948 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/search/SearchAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.search; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.Client; + +/** + */ +public class SearchAction extends Action { + + public static final SearchAction INSTANCE = new SearchAction(); + public static final String NAME = "search"; + + private SearchAction() { + super(NAME); + } + + @Override + public SearchResponse newResponse() { + return new SearchResponse(); + } + + @Override + public SearchRequestBuilder newRequestBuilder(Client client) { + return new SearchRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/src/main/java/org/elasticsearch/action/search/SearchResponse.java index e5a59d3cdac..cbdf347dbf5 100644 --- a/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -39,8 +39,6 @@ import static org.elasticsearch.search.internal.InternalSearchResponse.readInter /** * A response of a search request. - * - * */ public class SearchResponse implements ActionResponse, ToXContent { @@ -56,7 +54,7 @@ public class SearchResponse implements ActionResponse, ToXContent { private long tookInMillis; - SearchResponse() { + public SearchResponse() { } public SearchResponse(InternalSearchResponse internalResponse, String scrollId, int totalShards, int successfulShards, long tookInMillis, ShardSearchFailure[] shardFailures) { diff --git a/src/main/java/org/elasticsearch/action/search/SearchScrollAction.java b/src/main/java/org/elasticsearch/action/search/SearchScrollAction.java new file mode 100644 index 00000000000..69f71679963 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/search/SearchScrollAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.search; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.Client; + +/** + */ +public class SearchScrollAction extends Action { + + public static final SearchScrollAction INSTANCE = new SearchScrollAction(); + public static final String NAME = "search/scroll"; + + private SearchScrollAction() { + super(NAME); + } + + @Override + public SearchResponse newResponse() { + return new SearchResponse(); + } + + @Override + public SearchScrollRequestBuilder newRequestBuilder(Client client) { + return new SearchScrollRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java b/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java index c70b3f417bc..bcfb83dab90 100644 --- a/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java +++ b/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java @@ -98,6 +98,11 @@ public class SearchScrollRequest implements ActionRequest { return scrollId; } + public SearchScrollRequest scrollId(String scrollId) { + this.scrollId = scrollId; + return this; + } + /** * If set, will enable scrolling of the search request. */ diff --git a/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java b/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java index 4d92029ed56..4e12d64aa33 100644 --- a/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java @@ -30,6 +30,10 @@ import org.elasticsearch.search.Scroll; */ public class SearchScrollRequestBuilder extends BaseRequestBuilder { + public SearchScrollRequestBuilder(Client client) { + super(client, new SearchScrollRequest()); + } + public SearchScrollRequestBuilder(Client client, String scrollId) { super(client, new SearchScrollRequest(scrollId)); } @@ -50,6 +54,14 @@ public class SearchScrollRequestBuilder extends BaseRequestBuilder { + + public static final UpdateAction INSTANCE = new UpdateAction(); + public static final String NAME = "update"; + + private UpdateAction() { + super(NAME); + } + + @Override + public UpdateResponse newResponse() { + return new UpdateResponse(); + } + + @Override + public UpdateRequestBuilder newRequestBuilder(Client client) { + return new UpdateRequestBuilder(client); + } +} diff --git a/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java index 6b19d66ce83..ed5b1ec8513 100644 --- a/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java @@ -32,6 +32,10 @@ import java.util.Map; */ public class UpdateRequestBuilder extends BaseRequestBuilder { + public UpdateRequestBuilder(Client client) { + super(client, new UpdateRequest()); + } + public UpdateRequestBuilder(Client client, String index, String type, String id) { super(client, new UpdateRequest(index, type, id)); } diff --git a/src/main/java/org/elasticsearch/client/Client.java b/src/main/java/org/elasticsearch/client/Client.java index 6e019eab6d1..44d413c64a7 100644 --- a/src/main/java/org/elasticsearch/client/Client.java +++ b/src/main/java/org/elasticsearch/client/Client.java @@ -19,8 +19,7 @@ package org.elasticsearch.client; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.*; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; @@ -73,6 +72,42 @@ public interface Client { */ AdminClient admin(); + /** + * Executes a generic action, denoted by an {@link Action}. + * + * @param action The action type to execute. + * @param request The action request. + * @param Teh request type. + * @param the response type. + * @param The request builder type. + * @return A future allowing to get back the response. + */ + > ActionFuture execute(final Action action, final Request request); + + /** + * Executes a generic action, denoted by an {@link Action}. + * + * @param action The action type to execute. + * @param request Teh action request. + * @param listener The listener to receive the response back. + * @param The request type. + * @param The response type. + * @param The request builder type. + */ + > void execute(final Action action, final Request request, ActionListener listener); + + /** + * Prepares a request builder to execute, specified by {@link Action}. + * + * @param action The action type to execute. + * @param The request type. + * @param The response type. + * @param The request builder. + * @return The request builder, that can, at a later stage, execute the request. + */ + > RequestBuilder prepareExecute(final Action action); + + /** * Index a JSON source associated with a given index and type. *

    diff --git a/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/src/main/java/org/elasticsearch/client/ClusterAdminClient.java index 3784ba2f564..114a2eedf81 100644 --- a/src/main/java/org/elasticsearch/client/ClusterAdminClient.java +++ b/src/main/java/org/elasticsearch/client/ClusterAdminClient.java @@ -19,8 +19,8 @@ package org.elasticsearch.client; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.*; +import org.elasticsearch.action.admin.cluster.ClusterAction; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; @@ -36,15 +36,6 @@ import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownRespons import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingRequest; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingRequestBuilder; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingResponse; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingRequest; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingRequestBuilder; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingResponse; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingRequest; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingRequestBuilder; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingResponse; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequestBuilder; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; @@ -62,6 +53,12 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; */ public interface ClusterAdminClient { + > ActionFuture execute(final ClusterAction action, final Request request); + + > void execute(final ClusterAction action, final Request request, ActionListener listener); + + > RequestBuilder prepareExecute(final ClusterAction action); + /** * The health of the cluster. * @@ -229,24 +226,4 @@ public interface ClusterAdminClient { * Restarts nodes in the cluster. */ NodesRestartRequestBuilder prepareNodesRestart(String... nodesIds); - - ActionFuture ping(SinglePingRequest request); - - void ping(SinglePingRequest request, ActionListener listener); - - SinglePingRequestBuilder preparePingSingle(); - - SinglePingRequestBuilder preparePingSingle(String index, String type, String id); - - ActionFuture ping(BroadcastPingRequest request); - - void ping(BroadcastPingRequest request, ActionListener listener); - - BroadcastPingRequestBuilder preparePingBroadcast(String... indices); - - ActionFuture ping(ReplicationPingRequest request); - - void ping(ReplicationPingRequest request, ActionListener listener); - - ReplicationPingRequestBuilder preparePingReplication(String... indices); } diff --git a/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/src/main/java/org/elasticsearch/client/IndicesAdminClient.java index 1ee20f70650..4e852109aa8 100644 --- a/src/main/java/org/elasticsearch/client/IndicesAdminClient.java +++ b/src/main/java/org/elasticsearch/client/IndicesAdminClient.java @@ -19,8 +19,8 @@ package org.elasticsearch.client; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.*; +import org.elasticsearch.action.admin.indices.IndicesAction; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; @@ -93,6 +93,13 @@ import org.elasticsearch.common.Nullable; */ public interface IndicesAdminClient { + > ActionFuture execute(final IndicesAction action, final Request request); + + > void execute(final IndicesAction action, final Request request, ActionListener listener); + + > RequestBuilder prepareExecute(final IndicesAction action); + + /** * Indices Exists. * diff --git a/src/main/java/org/elasticsearch/client/Requests.java b/src/main/java/org/elasticsearch/client/Requests.java index e19bcc80654..ebe1f5f178d 100644 --- a/src/main/java/org/elasticsearch/client/Requests.java +++ b/src/main/java/org/elasticsearch/client/Requests.java @@ -24,9 +24,6 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartRequest; import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownRequest; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingRequest; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingRequest; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingRequest; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; @@ -59,8 +56,6 @@ import org.elasticsearch.common.xcontent.XContentType; /** * A handy one stop shop for creating requests (make sure to import static this class). - * - * */ public class Requests { @@ -441,16 +436,4 @@ public class Requests { public static NodesRestartRequest nodesRestartRequest(String... nodesIds) { return new NodesRestartRequest(nodesIds); } - - public static SinglePingRequest pingSingleRequest(String index) { - return new SinglePingRequest(index); - } - - public static BroadcastPingRequest pingBroadcastRequest(String... indices) { - return new BroadcastPingRequest(indices); - } - - public static ReplicationPingRequest pingReplicationRequest(String... indices) { - return new ReplicationPingRequest(indices); - } } diff --git a/src/main/java/org/elasticsearch/client/node/NodeClient.java b/src/main/java/org/elasticsearch/client/node/NodeClient.java index 2de1400df69..a9489f0de8a 100644 --- a/src/main/java/org/elasticsearch/client/node/NodeClient.java +++ b/src/main/java/org/elasticsearch/client/node/NodeClient.java @@ -19,38 +19,13 @@ package org.elasticsearch.client.node; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.bulk.TransportBulkAction; -import org.elasticsearch.action.count.CountRequest; -import org.elasticsearch.action.count.CountResponse; -import org.elasticsearch.action.count.TransportCountAction; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.delete.DeleteResponse; -import org.elasticsearch.action.delete.TransportDeleteAction; -import org.elasticsearch.action.deletebyquery.DeleteByQueryRequest; -import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse; -import org.elasticsearch.action.deletebyquery.TransportDeleteByQueryAction; -import org.elasticsearch.action.get.*; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.index.TransportIndexAction; -import org.elasticsearch.action.mlt.MoreLikeThisRequest; -import org.elasticsearch.action.mlt.TransportMoreLikeThisAction; -import org.elasticsearch.action.percolate.PercolateRequest; -import org.elasticsearch.action.percolate.PercolateResponse; -import org.elasticsearch.action.percolate.TransportPercolateAction; -import org.elasticsearch.action.search.*; +import com.google.common.collect.ImmutableMap; +import org.elasticsearch.action.*; import org.elasticsearch.action.support.TransportAction; -import org.elasticsearch.action.update.TransportUpdateAction; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.internal.InternalClient; import org.elasticsearch.client.support.AbstractClient; +import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; @@ -66,46 +41,19 @@ public class NodeClient extends AbstractClient implements InternalClient { private final NodeAdminClient admin; - private final TransportIndexAction indexAction; - - private final TransportUpdateAction updateAction; - - private final TransportDeleteAction deleteAction; - - private final TransportBulkAction bulkAction; - - private final TransportDeleteByQueryAction deleteByQueryAction; - - private final TransportGetAction getAction; - - private final TransportMultiGetAction multiGetAction; - - private final TransportCountAction countAction; - - private final TransportSearchAction searchAction; - - private final TransportSearchScrollAction searchScrollAction; - - private final TransportMoreLikeThisAction moreLikeThisAction; - - private final TransportPercolateAction percolateAction; + private final ImmutableMap actions; @Inject - public NodeClient(Settings settings, ThreadPool threadPool, NodeAdminClient admin, Map actions) { + public NodeClient(Settings settings, ThreadPool threadPool, NodeAdminClient admin, Map actions) { this.threadPool = threadPool; this.admin = admin; - this.indexAction = (TransportIndexAction) actions.get(TransportActions.INDEX); - this.updateAction = (TransportUpdateAction) actions.get(TransportActions.UPDATE); - this.deleteAction = (TransportDeleteAction) actions.get(TransportActions.DELETE); - this.bulkAction = (TransportBulkAction) actions.get(TransportActions.BULK); - this.deleteByQueryAction = (TransportDeleteByQueryAction) actions.get(TransportActions.DELETE_BY_QUERY); - this.getAction = (TransportGetAction) actions.get(TransportActions.GET); - this.multiGetAction = (TransportMultiGetAction) actions.get(TransportActions.MULTI_GET); - this.countAction = (TransportCountAction) actions.get(TransportActions.COUNT); - this.searchAction = (TransportSearchAction) actions.get(TransportActions.SEARCH); - this.searchScrollAction = (TransportSearchScrollAction) actions.get(TransportActions.SEARCH_SCROLL); - this.moreLikeThisAction = (TransportMoreLikeThisAction) actions.get(TransportActions.MORE_LIKE_THIS); - this.percolateAction = (TransportPercolateAction) actions.get(TransportActions.PERCOLATE); + MapBuilder actionsBuilder = new MapBuilder(); + for (Map.Entry entry : actions.entrySet()) { + if (entry.getKey() instanceof Action) { + actionsBuilder.put((Action) entry.getKey(), entry.getValue()); + } + } + this.actions = actionsBuilder.immutableMap(); } @Override @@ -124,122 +72,14 @@ public class NodeClient extends AbstractClient implements InternalClient { } @Override - public ActionFuture index(IndexRequest request) { - return indexAction.execute(request); + public > ActionFuture execute(Action action, Request request) { + TransportAction transportAction = actions.get(action); + return transportAction.execute(request); } @Override - public void index(IndexRequest request, ActionListener listener) { - indexAction.execute(request, listener); - } - - @Override - public ActionFuture update(UpdateRequest request) { - return updateAction.execute(request); - } - - @Override - public void update(UpdateRequest request, ActionListener listener) { - updateAction.execute(request, listener); - } - - @Override - public ActionFuture delete(DeleteRequest request) { - return deleteAction.execute(request); - } - - @Override - public void delete(DeleteRequest request, ActionListener listener) { - deleteAction.execute(request, listener); - } - - @Override - public ActionFuture bulk(BulkRequest request) { - return bulkAction.execute(request); - } - - @Override - public void bulk(BulkRequest request, ActionListener listener) { - bulkAction.execute(request, listener); - } - - @Override - public ActionFuture deleteByQuery(DeleteByQueryRequest request) { - return deleteByQueryAction.execute(request); - } - - @Override - public void deleteByQuery(DeleteByQueryRequest request, ActionListener listener) { - deleteByQueryAction.execute(request, listener); - } - - @Override - public ActionFuture get(GetRequest request) { - return getAction.execute(request); - } - - @Override - public void get(GetRequest request, ActionListener listener) { - getAction.execute(request, listener); - } - - @Override - public ActionFuture multiGet(MultiGetRequest request) { - return multiGetAction.execute(request); - } - - @Override - public void multiGet(MultiGetRequest request, ActionListener listener) { - multiGetAction.execute(request, listener); - } - - @Override - public ActionFuture count(CountRequest request) { - return countAction.execute(request); - } - - @Override - public void count(CountRequest request, ActionListener listener) { - countAction.execute(request, listener); - } - - @Override - public ActionFuture search(SearchRequest request) { - return searchAction.execute(request); - } - - @Override - public void search(SearchRequest request, ActionListener listener) { - searchAction.execute(request, listener); - } - - @Override - public ActionFuture searchScroll(SearchScrollRequest request) { - return searchScrollAction.execute(request); - } - - @Override - public void searchScroll(SearchScrollRequest request, ActionListener listener) { - searchScrollAction.execute(request, listener); - } - - @Override - public ActionFuture moreLikeThis(MoreLikeThisRequest request) { - return moreLikeThisAction.execute(request); - } - - @Override - public void moreLikeThis(MoreLikeThisRequest request, ActionListener listener) { - moreLikeThisAction.execute(request, listener); - } - - @Override - public ActionFuture percolate(PercolateRequest request) { - return percolateAction.execute(request); - } - - @Override - public void percolate(PercolateRequest request, ActionListener listener) { - percolateAction.execute(request, listener); + public > void execute(Action action, Request request, ActionListener listener) { + TransportAction transportAction = actions.get(action); + transportAction.execute(request, listener); } } diff --git a/src/main/java/org/elasticsearch/client/node/NodeClusterAdminClient.java b/src/main/java/org/elasticsearch/client/node/NodeClusterAdminClient.java index d0f81e75340..5dd880698f3 100644 --- a/src/main/java/org/elasticsearch/client/node/NodeClusterAdminClient.java +++ b/src/main/java/org/elasticsearch/client/node/NodeClusterAdminClient.java @@ -19,45 +19,13 @@ package org.elasticsearch.client.node; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; -import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartRequest; -import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartResponse; -import org.elasticsearch.action.admin.cluster.node.restart.TransportNodesRestartAction; -import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownRequest; -import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownResponse; -import org.elasticsearch.action.admin.cluster.node.shutdown.TransportNodesShutdownAction; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingRequest; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingResponse; -import org.elasticsearch.action.admin.cluster.ping.broadcast.TransportBroadcastPingAction; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingRequest; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingResponse; -import org.elasticsearch.action.admin.cluster.ping.replication.TransportReplicationPingAction; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingRequest; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingResponse; -import org.elasticsearch.action.admin.cluster.ping.single.TransportSinglePingAction; -import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest; -import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; -import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction; -import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; -import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; -import org.elasticsearch.action.admin.cluster.settings.TransportClusterUpdateSettingsAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.action.admin.cluster.state.TransportClusterStateAction; +import com.google.common.collect.ImmutableMap; +import org.elasticsearch.action.*; +import org.elasticsearch.action.admin.cluster.ClusterAction; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.internal.InternalClusterAdminClient; import org.elasticsearch.client.support.AbstractClusterAdminClient; +import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; @@ -71,42 +39,18 @@ public class NodeClusterAdminClient extends AbstractClusterAdminClient implement private final ThreadPool threadPool; - private final TransportClusterHealthAction clusterHealthAction; - - private final TransportClusterStateAction clusterStateAction; - - private final TransportClusterRerouteAction clusterRerouteAction; - - private final TransportClusterUpdateSettingsAction clusterUpdateSettingsAction; - - private final TransportSinglePingAction singlePingAction; - - private final TransportBroadcastPingAction broadcastPingAction; - - private final TransportReplicationPingAction replicationPingAction; - - private final TransportNodesInfoAction nodesInfoAction; - - private final TransportNodesStatsAction nodesStatsAction; - - private final TransportNodesShutdownAction nodesShutdown; - - private final TransportNodesRestartAction nodesRestart; + private final ImmutableMap actions; @Inject - public NodeClusterAdminClient(Settings settings, ThreadPool threadPool, Map actions) { + public NodeClusterAdminClient(Settings settings, ThreadPool threadPool, Map actions) { this.threadPool = threadPool; - this.clusterRerouteAction = (TransportClusterRerouteAction) actions.get(TransportActions.Admin.Cluster.REROUTE); - this.clusterHealthAction = (TransportClusterHealthAction) actions.get(TransportActions.Admin.Cluster.HEALTH); - this.clusterStateAction = (TransportClusterStateAction) actions.get(TransportActions.Admin.Cluster.STATE); - this.clusterUpdateSettingsAction = (TransportClusterUpdateSettingsAction) actions.get(TransportActions.Admin.Cluster.UPDATE_SETTINGS); - this.nodesInfoAction = (TransportNodesInfoAction) actions.get(TransportActions.Admin.Cluster.Node.INFO); - this.nodesStatsAction = (TransportNodesStatsAction) actions.get(TransportActions.Admin.Cluster.Node.STATS); - this.nodesShutdown = (TransportNodesShutdownAction) actions.get(TransportActions.Admin.Cluster.Node.SHUTDOWN); - this.nodesRestart = (TransportNodesRestartAction) actions.get(TransportActions.Admin.Cluster.Node.RESTART); - this.singlePingAction = (TransportSinglePingAction) actions.get(TransportActions.Admin.Cluster.Ping.SINGLE); - this.broadcastPingAction = (TransportBroadcastPingAction) actions.get(TransportActions.Admin.Cluster.Ping.BROADCAST); - this.replicationPingAction = (TransportReplicationPingAction) actions.get(TransportActions.Admin.Cluster.Ping.REPLICATION); + MapBuilder actionsBuilder = new MapBuilder(); + for (Map.Entry entry : actions.entrySet()) { + if (entry.getKey() instanceof ClusterAction) { + actionsBuilder.put((ClusterAction) entry.getKey(), entry.getValue()); + } + } + this.actions = actionsBuilder.immutableMap(); } @Override @@ -115,112 +59,14 @@ public class NodeClusterAdminClient extends AbstractClusterAdminClient implement } @Override - public ActionFuture health(ClusterHealthRequest request) { - return clusterHealthAction.execute(request); + public > ActionFuture execute(ClusterAction action, Request request) { + TransportAction transportAction = actions.get(action); + return transportAction.execute(request); } @Override - public void health(ClusterHealthRequest request, ActionListener listener) { - clusterHealthAction.execute(request, listener); - } - - @Override - public ActionFuture state(ClusterStateRequest request) { - return clusterStateAction.execute(request); - } - - @Override - public void state(ClusterStateRequest request, ActionListener listener) { - clusterStateAction.execute(request, listener); - } - - @Override - public ActionFuture reroute(ClusterRerouteRequest request) { - return clusterRerouteAction.execute(request); - } - - @Override - public void reroute(ClusterRerouteRequest request, ActionListener listener) { - clusterRerouteAction.execute(request, listener); - } - - @Override - public ActionFuture updateSettings(ClusterUpdateSettingsRequest request) { - return clusterUpdateSettingsAction.execute(request); - } - - @Override - public void updateSettings(ClusterUpdateSettingsRequest request, ActionListener listener) { - clusterUpdateSettingsAction.execute(request, listener); - } - - @Override - public ActionFuture ping(SinglePingRequest request) { - return singlePingAction.execute(request); - } - - @Override - public void ping(SinglePingRequest request, ActionListener listener) { - singlePingAction.execute(request, listener); - } - - @Override - public ActionFuture ping(BroadcastPingRequest request) { - return broadcastPingAction.execute(request); - } - - @Override - public void ping(BroadcastPingRequest request, ActionListener listener) { - broadcastPingAction.execute(request, listener); - } - - @Override - public ActionFuture ping(ReplicationPingRequest request) { - return replicationPingAction.execute(request); - } - - @Override - public void ping(ReplicationPingRequest request, ActionListener listener) { - replicationPingAction.execute(request, listener); - } - - @Override - public ActionFuture nodesInfo(NodesInfoRequest request) { - return nodesInfoAction.execute(request); - } - - @Override - public void nodesInfo(NodesInfoRequest request, ActionListener listener) { - nodesInfoAction.execute(request, listener); - } - - @Override - public ActionFuture nodesStats(NodesStatsRequest request) { - return nodesStatsAction.execute(request); - } - - @Override - public void nodesStats(NodesStatsRequest request, ActionListener listener) { - nodesStatsAction.execute(request, listener); - } - - @Override - public ActionFuture nodesShutdown(NodesShutdownRequest request) { - return nodesShutdown.execute(request); - } - - @Override - public void nodesShutdown(NodesShutdownRequest request, ActionListener listener) { - nodesShutdown.execute(request, listener); - } - - @Override - public ActionFuture nodesRestart(NodesRestartRequest request) { - return nodesRestart.execute(request); - } - - @Override - public void nodesRestart(NodesRestartRequest request, ActionListener listener) { - nodesRestart.execute(request, listener); + public > void execute(ClusterAction action, Request request, ActionListener listener) { + TransportAction transportAction = actions.get(action); + transportAction.execute(request, listener); } } diff --git a/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java b/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java index c69ea9b9f48..cb75b4f7029 100644 --- a/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java +++ b/src/main/java/org/elasticsearch/client/node/NodeIndicesAdminClient.java @@ -19,75 +19,13 @@ package org.elasticsearch.client.node; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; -import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; -import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; -import org.elasticsearch.action.admin.indices.cache.clear.TransportClearIndicesCacheAction; -import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; -import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; -import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; -import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction; -import org.elasticsearch.action.admin.indices.exists.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.IndicesExistsResponse; -import org.elasticsearch.action.admin.indices.exists.TransportIndicesExistsAction; -import org.elasticsearch.action.admin.indices.flush.FlushRequest; -import org.elasticsearch.action.admin.indices.flush.FlushResponse; -import org.elasticsearch.action.admin.indices.flush.TransportFlushAction; -import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotRequest; -import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotResponse; -import org.elasticsearch.action.admin.indices.gateway.snapshot.TransportGatewaySnapshotAction; -import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingResponse; -import org.elasticsearch.action.admin.indices.mapping.delete.TransportDeleteMappingAction; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; -import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction; -import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; -import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; -import org.elasticsearch.action.admin.indices.open.TransportOpenIndexAction; -import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest; -import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; -import org.elasticsearch.action.admin.indices.optimize.TransportOptimizeAction; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.action.admin.indices.refresh.TransportRefreshAction; -import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; -import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest; -import org.elasticsearch.action.admin.indices.segments.TransportIndicesSegmentsAction; -import org.elasticsearch.action.admin.indices.settings.TransportUpdateSettingsAction; -import org.elasticsearch.action.admin.indices.settings.UpdateSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.UpdateSettingsResponse; -import org.elasticsearch.action.admin.indices.stats.IndicesStats; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; -import org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction; -import org.elasticsearch.action.admin.indices.status.IndicesStatusRequest; -import org.elasticsearch.action.admin.indices.status.IndicesStatusResponse; -import org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction; -import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse; -import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteIndexTemplateAction; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; -import org.elasticsearch.action.admin.indices.template.put.TransportPutIndexTemplateAction; -import org.elasticsearch.action.admin.indices.validate.query.TransportValidateQueryAction; -import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; -import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; +import com.google.common.collect.ImmutableMap; +import org.elasticsearch.action.*; +import org.elasticsearch.action.admin.indices.IndicesAction; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.client.support.AbstractIndicesAdminClient; +import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; @@ -101,72 +39,18 @@ public class NodeIndicesAdminClient extends AbstractIndicesAdminClient implement private final ThreadPool threadPool; - private final TransportIndicesExistsAction indicesExistsAction; - - private final TransportIndicesStatsAction indicesStatsAction; - - private final TransportIndicesStatusAction indicesStatusAction; - - private final TransportIndicesSegmentsAction indicesSegmentsAction; - - private final TransportCreateIndexAction createIndexAction; - - private final TransportDeleteIndexAction deleteIndexAction; - - private final TransportCloseIndexAction closeIndexAction; - - private final TransportOpenIndexAction openIndexAction; - - private final TransportRefreshAction refreshAction; - - private final TransportFlushAction flushAction; - - private final TransportOptimizeAction optimizeAction; - - private final TransportPutMappingAction putMappingAction; - - private final TransportDeleteMappingAction deleteMappingAction; - - private final TransportGatewaySnapshotAction gatewaySnapshotAction; - - private final TransportIndicesAliasesAction indicesAliasesAction; - - private final TransportClearIndicesCacheAction clearIndicesCacheAction; - - private final TransportUpdateSettingsAction updateSettingsAction; - - private final TransportAnalyzeAction analyzeAction; - - private final TransportPutIndexTemplateAction putIndexTemplateAction; - - private final TransportDeleteIndexTemplateAction deleteIndexTemplateAction; - - private final TransportValidateQueryAction validateQueryAction; + private final ImmutableMap actions; @Inject - public NodeIndicesAdminClient(Settings settings, ThreadPool threadPool, Map actions) { + public NodeIndicesAdminClient(Settings settings, ThreadPool threadPool, Map actions) { this.threadPool = threadPool; - this.indicesExistsAction = (TransportIndicesExistsAction) actions.get(TransportActions.Admin.Indices.EXISTS); - this.indicesStatsAction = (TransportIndicesStatsAction) actions.get(TransportActions.Admin.Indices.STATS); - this.indicesStatusAction = (TransportIndicesStatusAction) actions.get(TransportActions.Admin.Indices.STATUS); - this.indicesSegmentsAction = (TransportIndicesSegmentsAction) actions.get(TransportActions.Admin.Indices.SEGMENTS); - this.createIndexAction = (TransportCreateIndexAction) actions.get(TransportActions.Admin.Indices.CREATE); - this.deleteIndexAction = (TransportDeleteIndexAction) actions.get(TransportActions.Admin.Indices.DELETE); - this.closeIndexAction = (TransportCloseIndexAction) actions.get(TransportActions.Admin.Indices.CLOSE); - this.openIndexAction = (TransportOpenIndexAction) actions.get(TransportActions.Admin.Indices.OPEN); - this.refreshAction = (TransportRefreshAction) actions.get(TransportActions.Admin.Indices.REFRESH); - this.flushAction = (TransportFlushAction) actions.get(TransportActions.Admin.Indices.FLUSH); - this.optimizeAction = (TransportOptimizeAction) actions.get(TransportActions.Admin.Indices.OPTIMIZE); - this.deleteMappingAction = (TransportDeleteMappingAction) actions.get(TransportActions.Admin.Indices.Mapping.DELETE); - this.putMappingAction = (TransportPutMappingAction) actions.get(TransportActions.Admin.Indices.Mapping.PUT); - this.gatewaySnapshotAction = (TransportGatewaySnapshotAction) actions.get(TransportActions.Admin.Indices.Gateway.SNAPSHOT); - this.indicesAliasesAction = (TransportIndicesAliasesAction) actions.get(TransportActions.Admin.Indices.ALIASES); - this.clearIndicesCacheAction = (TransportClearIndicesCacheAction) actions.get(TransportActions.Admin.Indices.Cache.CLEAR); - this.updateSettingsAction = (TransportUpdateSettingsAction) actions.get(TransportActions.Admin.Indices.UPDATE_SETTINGS); - this.analyzeAction = (TransportAnalyzeAction) actions.get(TransportActions.Admin.Indices.ANALYZE); - this.putIndexTemplateAction = (TransportPutIndexTemplateAction) actions.get(TransportActions.Admin.Indices.Template.PUT); - this.deleteIndexTemplateAction = (TransportDeleteIndexTemplateAction) actions.get(TransportActions.Admin.Indices.Template.DELETE); - this.validateQueryAction = (TransportValidateQueryAction) actions.get(TransportActions.Admin.Indices.Validate.QUERY); + MapBuilder actionsBuilder = new MapBuilder(); + for (Map.Entry entry : actions.entrySet()) { + if (entry.getKey() instanceof IndicesAction) { + actionsBuilder.put((IndicesAction) entry.getKey(), entry.getValue()); + } + } + this.actions = actionsBuilder.immutableMap(); } @Override @@ -175,212 +59,14 @@ public class NodeIndicesAdminClient extends AbstractIndicesAdminClient implement } @Override - public ActionFuture exists(IndicesExistsRequest request) { - return indicesExistsAction.execute(request); + public > ActionFuture execute(IndicesAction action, Request request) { + TransportAction transportAction = actions.get(action); + return transportAction.execute(request); } @Override - public void exists(IndicesExistsRequest request, ActionListener listener) { - indicesExistsAction.execute(request, listener); - } - - @Override - public ActionFuture stats(IndicesStatsRequest request) { - return indicesStatsAction.execute(request); - } - - @Override - public void stats(IndicesStatsRequest request, ActionListener lister) { - indicesStatsAction.execute(request, lister); - } - - @Override - public ActionFuture status(IndicesStatusRequest request) { - return indicesStatusAction.execute(request); - } - - @Override - public void status(IndicesStatusRequest request, ActionListener listener) { - indicesStatusAction.execute(request, listener); - } - - @Override - public ActionFuture segments(IndicesSegmentsRequest request) { - return indicesSegmentsAction.execute(request); - } - - @Override - public void segments(IndicesSegmentsRequest request, ActionListener listener) { - indicesSegmentsAction.execute(request, listener); - } - - @Override - public ActionFuture create(CreateIndexRequest request) { - return createIndexAction.execute(request); - } - - @Override - public void create(CreateIndexRequest request, ActionListener listener) { - createIndexAction.execute(request, listener); - } - - @Override - public ActionFuture delete(DeleteIndexRequest request) { - return deleteIndexAction.execute(request); - } - - @Override - public void delete(DeleteIndexRequest request, ActionListener listener) { - deleteIndexAction.execute(request, listener); - } - - @Override - public ActionFuture close(CloseIndexRequest request) { - return closeIndexAction.execute(request); - } - - @Override - public void close(CloseIndexRequest request, ActionListener listener) { - closeIndexAction.execute(request, listener); - } - - @Override - public ActionFuture open(OpenIndexRequest request) { - return openIndexAction.execute(request); - } - - @Override - public void open(OpenIndexRequest request, ActionListener listener) { - openIndexAction.execute(request, listener); - } - - @Override - public ActionFuture refresh(RefreshRequest request) { - return refreshAction.execute(request); - } - - @Override - public void refresh(RefreshRequest request, ActionListener listener) { - refreshAction.execute(request, listener); - } - - @Override - public ActionFuture flush(FlushRequest request) { - return flushAction.execute(request); - } - - @Override - public void flush(FlushRequest request, ActionListener listener) { - flushAction.execute(request, listener); - } - - @Override - public ActionFuture optimize(OptimizeRequest request) { - return optimizeAction.execute(request); - } - - @Override - public void optimize(OptimizeRequest request, ActionListener listener) { - optimizeAction.execute(request, listener); - } - - @Override - public ActionFuture putMapping(PutMappingRequest request) { - return putMappingAction.execute(request); - } - - @Override - public void putMapping(PutMappingRequest request, ActionListener listener) { - putMappingAction.execute(request, listener); - } - - @Override - public ActionFuture deleteMapping(DeleteMappingRequest request) { - return deleteMappingAction.execute(request); - } - - @Override - public void deleteMapping(DeleteMappingRequest request, ActionListener listener) { - deleteMappingAction.execute(request, listener); - } - - @Override - public ActionFuture gatewaySnapshot(GatewaySnapshotRequest request) { - return gatewaySnapshotAction.execute(request); - } - - @Override - public void gatewaySnapshot(GatewaySnapshotRequest request, ActionListener listener) { - gatewaySnapshotAction.execute(request, listener); - } - - @Override - public ActionFuture aliases(IndicesAliasesRequest request) { - return indicesAliasesAction.execute(request); - } - - @Override - public void aliases(IndicesAliasesRequest request, ActionListener listener) { - indicesAliasesAction.execute(request, listener); - } - - @Override - public ActionFuture clearCache(ClearIndicesCacheRequest request) { - return clearIndicesCacheAction.execute(request); - } - - @Override - public void clearCache(ClearIndicesCacheRequest request, ActionListener listener) { - clearIndicesCacheAction.execute(request, listener); - } - - @Override - public ActionFuture updateSettings(UpdateSettingsRequest request) { - return updateSettingsAction.execute(request); - } - - @Override - public void updateSettings(UpdateSettingsRequest request, ActionListener listener) { - updateSettingsAction.execute(request, listener); - } - - @Override - public ActionFuture analyze(AnalyzeRequest request) { - return analyzeAction.execute(request); - } - - @Override - public void analyze(AnalyzeRequest request, ActionListener listener) { - analyzeAction.execute(request, listener); - } - - @Override - public ActionFuture putTemplate(PutIndexTemplateRequest request) { - return putIndexTemplateAction.execute(request); - } - - @Override - public void putTemplate(PutIndexTemplateRequest request, ActionListener listener) { - putIndexTemplateAction.execute(request, listener); - } - - @Override - public ActionFuture deleteTemplate(DeleteIndexTemplateRequest request) { - return deleteIndexTemplateAction.execute(request); - } - - @Override - public void deleteTemplate(DeleteIndexTemplateRequest request, ActionListener listener) { - deleteIndexTemplateAction.execute(request, listener); - } - - @Override - public ActionFuture validateQuery(ValidateQueryRequest request) { - return validateQueryAction.execute(request); - } - - @Override - public void validateQuery(ValidateQueryRequest request, ActionListener listener) { - validateQueryAction.execute(request, listener); + public > void execute(IndicesAction action, Request request, ActionListener listener) { + TransportAction transportAction = actions.get(action); + transportAction.execute(request, listener); } } diff --git a/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/src/main/java/org/elasticsearch/client/support/AbstractClient.java index add13aa15b7..41e06f01ce6 100644 --- a/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -19,18 +19,40 @@ package org.elasticsearch.client.support; +import org.elasticsearch.action.*; +import org.elasticsearch.action.bulk.BulkAction; +import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.count.CountAction; +import org.elasticsearch.action.count.CountRequest; import org.elasticsearch.action.count.CountRequestBuilder; +import org.elasticsearch.action.count.CountResponse; +import org.elasticsearch.action.delete.DeleteAction; +import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteRequestBuilder; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.deletebyquery.DeleteByQueryAction; +import org.elasticsearch.action.deletebyquery.DeleteByQueryRequest; import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder; -import org.elasticsearch.action.get.GetRequestBuilder; -import org.elasticsearch.action.get.MultiGetRequestBuilder; +import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse; +import org.elasticsearch.action.get.*; +import org.elasticsearch.action.index.IndexAction; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.mlt.MoreLikeThisAction; +import org.elasticsearch.action.mlt.MoreLikeThisRequest; import org.elasticsearch.action.mlt.MoreLikeThisRequestBuilder; +import org.elasticsearch.action.percolate.PercolateAction; +import org.elasticsearch.action.percolate.PercolateRequest; import org.elasticsearch.action.percolate.PercolateRequestBuilder; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.action.percolate.PercolateResponse; +import org.elasticsearch.action.search.*; +import org.elasticsearch.action.update.UpdateAction; +import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateRequestBuilder; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.InternalClient; import org.elasticsearch.common.Nullable; @@ -39,6 +61,21 @@ import org.elasticsearch.common.Nullable; */ public abstract class AbstractClient implements InternalClient { + @Override + public > RequestBuilder prepareExecute(final Action action) { + return action.newRequestBuilder(this); + } + + @Override + public ActionFuture index(final IndexRequest request) { + return execute(IndexAction.INSTANCE, request); + } + + @Override + public void index(final IndexRequest request, final ActionListener listener) { + execute(IndexAction.INSTANCE, request, listener); + } + @Override public IndexRequestBuilder prepareIndex() { return new IndexRequestBuilder(this, null); @@ -54,6 +91,16 @@ public abstract class AbstractClient implements InternalClient { return prepareIndex().setIndex(index).setType(type).setId(id); } + @Override + public ActionFuture update(final UpdateRequest request) { + return execute(UpdateAction.INSTANCE, request); + } + + @Override + public void update(final UpdateRequest request, final ActionListener listener) { + execute(UpdateAction.INSTANCE, request, listener); + } + @Override public UpdateRequestBuilder prepareUpdate() { return new UpdateRequestBuilder(this, null, null, null); @@ -64,6 +111,16 @@ public abstract class AbstractClient implements InternalClient { return new UpdateRequestBuilder(this, index, type, id); } + @Override + public ActionFuture delete(final DeleteRequest request) { + return execute(DeleteAction.INSTANCE, request); + } + + @Override + public void delete(final DeleteRequest request, final ActionListener listener) { + execute(DeleteAction.INSTANCE, request, listener); + } + @Override public DeleteRequestBuilder prepareDelete() { return new DeleteRequestBuilder(this, null); @@ -74,16 +131,46 @@ public abstract class AbstractClient implements InternalClient { return prepareDelete().setIndex(index).setType(type).setId(id); } + @Override + public ActionFuture bulk(final BulkRequest request) { + return execute(BulkAction.INSTANCE, request); + } + + @Override + public void bulk(final BulkRequest request, final ActionListener listener) { + execute(BulkAction.INSTANCE, request, listener); + } + @Override public BulkRequestBuilder prepareBulk() { return new BulkRequestBuilder(this); } + @Override + public ActionFuture deleteByQuery(final DeleteByQueryRequest request) { + return execute(DeleteByQueryAction.INSTANCE, request); + } + + @Override + public void deleteByQuery(final DeleteByQueryRequest request, final ActionListener listener) { + execute(DeleteByQueryAction.INSTANCE, request, listener); + } + @Override public DeleteByQueryRequestBuilder prepareDeleteByQuery(String... indices) { return new DeleteByQueryRequestBuilder(this).setIndices(indices); } + @Override + public ActionFuture get(final GetRequest request) { + return execute(GetAction.INSTANCE, request); + } + + @Override + public void get(final GetRequest request, final ActionListener listener) { + execute(GetAction.INSTANCE, request, listener); + } + @Override public GetRequestBuilder prepareGet() { return new GetRequestBuilder(this, null); @@ -94,31 +181,91 @@ public abstract class AbstractClient implements InternalClient { return prepareGet().setIndex(index).setType(type).setId(id); } + @Override + public ActionFuture multiGet(final MultiGetRequest request) { + return execute(MultiGetAction.INSTANCE, request); + } + + @Override + public void multiGet(final MultiGetRequest request, final ActionListener listener) { + execute(MultiGetAction.INSTANCE, request, listener); + } + @Override public MultiGetRequestBuilder prepareMultiGet() { return new MultiGetRequestBuilder(this); } + @Override + public ActionFuture search(final SearchRequest request) { + return execute(SearchAction.INSTANCE, request); + } + + @Override + public void search(final SearchRequest request, final ActionListener listener) { + execute(SearchAction.INSTANCE, request, listener); + } + @Override public SearchRequestBuilder prepareSearch(String... indices) { return new SearchRequestBuilder(this).setIndices(indices); } + @Override + public ActionFuture searchScroll(final SearchScrollRequest request) { + return execute(SearchScrollAction.INSTANCE, request); + } + + @Override + public void searchScroll(final SearchScrollRequest request, final ActionListener listener) { + execute(SearchScrollAction.INSTANCE, request, listener); + } + @Override public SearchScrollRequestBuilder prepareSearchScroll(String scrollId) { return new SearchScrollRequestBuilder(this, scrollId); } + @Override + public ActionFuture count(final CountRequest request) { + return execute(CountAction.INSTANCE, request); + } + + @Override + public void count(final CountRequest request, final ActionListener listener) { + execute(CountAction.INSTANCE, request, listener); + } + @Override public CountRequestBuilder prepareCount(String... indices) { return new CountRequestBuilder(this).setIndices(indices); } + @Override + public ActionFuture moreLikeThis(final MoreLikeThisRequest request) { + return execute(MoreLikeThisAction.INSTANCE, request); + } + + @Override + public void moreLikeThis(final MoreLikeThisRequest request, final ActionListener listener) { + execute(MoreLikeThisAction.INSTANCE, request, listener); + } + @Override public MoreLikeThisRequestBuilder prepareMoreLikeThis(String index, String type, String id) { return new MoreLikeThisRequestBuilder(this, index, type, id); } + @Override + public ActionFuture percolate(final PercolateRequest request) { + return execute(PercolateAction.INSTANCE, request); + } + + @Override + public void percolate(final PercolateRequest request, final ActionListener listener) { + execute(PercolateAction.INSTANCE, request, listener); + } + @Override public PercolateRequestBuilder preparePercolate(String index, String type) { return new PercolateRequestBuilder(this, index, type); diff --git a/src/main/java/org/elasticsearch/client/support/AbstractClusterAdminClient.java b/src/main/java/org/elasticsearch/client/support/AbstractClusterAdminClient.java index 2fce446801a..eb6561640a9 100644 --- a/src/main/java/org/elasticsearch/client/support/AbstractClusterAdminClient.java +++ b/src/main/java/org/elasticsearch/client/support/AbstractClusterAdminClient.java @@ -19,17 +19,40 @@ package org.elasticsearch.client.support; +import org.elasticsearch.action.*; +import org.elasticsearch.action.admin.cluster.ClusterAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequestBuilder; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartAction; +import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartRequest; import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartRequestBuilder; +import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartResponse; +import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownAction; +import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownRequest; import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownRequestBuilder; +import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownResponse; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingRequestBuilder; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingRequestBuilder; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingRequestBuilder; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteAction; +import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequestBuilder; +import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsAction; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequestBuilder; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.client.internal.InternalClusterAdminClient; /** @@ -37,63 +60,128 @@ import org.elasticsearch.client.internal.InternalClusterAdminClient; */ public abstract class AbstractClusterAdminClient implements InternalClusterAdminClient { + @Override + public > RequestBuilder prepareExecute(ClusterAction action) { + return action.newRequestBuilder(this); + } + + @Override + public ActionFuture health(final ClusterHealthRequest request) { + return execute(ClusterHealthAction.INSTANCE, request); + } + + @Override + public void health(final ClusterHealthRequest request, final ActionListener listener) { + execute(ClusterHealthAction.INSTANCE, request, listener); + } + @Override public ClusterHealthRequestBuilder prepareHealth(String... indices) { return new ClusterHealthRequestBuilder(this).setIndices(indices); } + @Override + public ActionFuture state(final ClusterStateRequest request) { + return execute(ClusterStateAction.INSTANCE, request); + } + + @Override + public void state(final ClusterStateRequest request, final ActionListener listener) { + execute(ClusterStateAction.INSTANCE, request, listener); + } + @Override public ClusterStateRequestBuilder prepareState() { return new ClusterStateRequestBuilder(this); } + @Override + public ActionFuture reroute(final ClusterRerouteRequest request) { + return execute(ClusterRerouteAction.INSTANCE, request); + } + + @Override + public void reroute(final ClusterRerouteRequest request, final ActionListener listener) { + execute(ClusterRerouteAction.INSTANCE, request, listener); + } + @Override public ClusterRerouteRequestBuilder prepareReroute() { return new ClusterRerouteRequestBuilder(this); } + @Override + public ActionFuture updateSettings(final ClusterUpdateSettingsRequest request) { + return execute(ClusterUpdateSettingsAction.INSTANCE, request); + } + + @Override + public void updateSettings(final ClusterUpdateSettingsRequest request, final ActionListener listener) { + execute(ClusterUpdateSettingsAction.INSTANCE, request, listener); + } + @Override public ClusterUpdateSettingsRequestBuilder prepareUpdateSettings() { return new ClusterUpdateSettingsRequestBuilder(this); } + @Override + public ActionFuture nodesInfo(final NodesInfoRequest request) { + return execute(NodesInfoAction.INSTANCE, request); + } + + @Override + public void nodesInfo(final NodesInfoRequest request, final ActionListener listener) { + execute(NodesInfoAction.INSTANCE, request, listener); + } + @Override public NodesInfoRequestBuilder prepareNodesInfo(String... nodesIds) { return new NodesInfoRequestBuilder(this).setNodesIds(nodesIds); } + @Override + public ActionFuture nodesStats(final NodesStatsRequest request) { + return execute(NodesStatsAction.INSTANCE, request); + } + + @Override + public void nodesStats(final NodesStatsRequest request, final ActionListener listener) { + execute(NodesStatsAction.INSTANCE, request, listener); + } + @Override public NodesStatsRequestBuilder prepareNodesStats(String... nodesIds) { return new NodesStatsRequestBuilder(this).setNodesIds(nodesIds); } + @Override + public ActionFuture nodesRestart(final NodesRestartRequest request) { + return execute(NodesRestartAction.INSTANCE, request); + } + + @Override + public void nodesRestart(final NodesRestartRequest request, final ActionListener listener) { + execute(NodesRestartAction.INSTANCE, request, listener); + } + @Override public NodesRestartRequestBuilder prepareNodesRestart(String... nodesIds) { return new NodesRestartRequestBuilder(this).setNodesIds(nodesIds); } + @Override + public ActionFuture nodesShutdown(final NodesShutdownRequest request) { + return execute(NodesShutdownAction.INSTANCE, request); + } + + @Override + public void nodesShutdown(final NodesShutdownRequest request, final ActionListener listener) { + execute(NodesShutdownAction.INSTANCE, request, listener); + } + @Override public NodesShutdownRequestBuilder prepareNodesShutdown(String... nodesIds) { return new NodesShutdownRequestBuilder(this).setNodesIds(nodesIds); } - - @Override - public SinglePingRequestBuilder preparePingSingle() { - return new SinglePingRequestBuilder(this); - } - - @Override - public SinglePingRequestBuilder preparePingSingle(String index, String type, String id) { - return preparePingSingle().setIndex(index).setType(type).setId(id); - } - - @Override - public BroadcastPingRequestBuilder preparePingBroadcast(String... indices) { - return new BroadcastPingRequestBuilder(this).setIndices(indices); - } - - @Override - public ReplicationPingRequestBuilder preparePingReplication(String... indices) { - return new ReplicationPingRequestBuilder(this).setIndices(indices); - } } diff --git a/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java b/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java index fb81126e924..61f16aa6a7f 100644 --- a/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java +++ b/src/main/java/org/elasticsearch/client/support/AbstractIndicesAdminClient.java @@ -19,27 +19,92 @@ package org.elasticsearch.client.support; +import org.elasticsearch.action.*; +import org.elasticsearch.action.admin.indices.IndicesAction; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheAction; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; +import org.elasticsearch.action.admin.indices.close.CloseIndexAction; +import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; +import org.elasticsearch.action.admin.indices.exists.IndicesExistsAction; +import org.elasticsearch.action.admin.indices.exists.IndicesExistsRequest; import org.elasticsearch.action.admin.indices.exists.IndicesExistsRequestBuilder; +import org.elasticsearch.action.admin.indices.exists.IndicesExistsResponse; +import org.elasticsearch.action.admin.indices.flush.FlushAction; +import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder; +import org.elasticsearch.action.admin.indices.flush.FlushResponse; +import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotAction; +import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotRequest; import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotRequestBuilder; +import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotResponse; +import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingAction; +import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequest; import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequestBuilder; +import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingResponse; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; +import org.elasticsearch.action.admin.indices.open.OpenIndexAction; +import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; +import org.elasticsearch.action.admin.indices.optimize.OptimizeAction; +import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest; import org.elasticsearch.action.admin.indices.optimize.OptimizeRequestBuilder; +import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; +import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsAction; +import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequestBuilder; +import org.elasticsearch.action.admin.indices.settings.UpdateSettingsAction; +import org.elasticsearch.action.admin.indices.settings.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.settings.UpdateSettingsRequestBuilder; +import org.elasticsearch.action.admin.indices.settings.UpdateSettingsResponse; +import org.elasticsearch.action.admin.indices.stats.IndicesStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; +import org.elasticsearch.action.admin.indices.status.IndicesStatusAction; +import org.elasticsearch.action.admin.indices.status.IndicesStatusRequest; import org.elasticsearch.action.admin.indices.status.IndicesStatusRequestBuilder; +import org.elasticsearch.action.admin.indices.status.IndicesStatusResponse; +import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder; +import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; +import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.elasticsearch.client.internal.InternalIndicesAdminClient; import org.elasticsearch.common.Nullable; @@ -48,91 +113,276 @@ import org.elasticsearch.common.Nullable; */ public abstract class AbstractIndicesAdminClient implements InternalIndicesAdminClient { + @Override + public > RequestBuilder prepareExecute(final IndicesAction action) { + return action.newRequestBuilder(this); + } + + @Override + public ActionFuture exists(final IndicesExistsRequest request) { + return execute(IndicesExistsAction.INSTANCE, request); + } + + @Override + public void exists(final IndicesExistsRequest request, final ActionListener listener) { + execute(IndicesExistsAction.INSTANCE, request, listener); + } + @Override public IndicesExistsRequestBuilder prepareExists(String... indices) { return new IndicesExistsRequestBuilder(this, indices); } + @Override + public ActionFuture aliases(final IndicesAliasesRequest request) { + return execute(IndicesAliasesAction.INSTANCE, request); + } + + @Override + public void aliases(final IndicesAliasesRequest request, final ActionListener listener) { + execute(IndicesAliasesAction.INSTANCE, request, listener); + } + @Override public IndicesAliasesRequestBuilder prepareAliases() { return new IndicesAliasesRequestBuilder(this); } + @Override + public ActionFuture clearCache(final ClearIndicesCacheRequest request) { + return execute(ClearIndicesCacheAction.INSTANCE, request); + } + + @Override + public void clearCache(final ClearIndicesCacheRequest request, final ActionListener listener) { + execute(ClearIndicesCacheAction.INSTANCE, request, listener); + } + @Override public ClearIndicesCacheRequestBuilder prepareClearCache(String... indices) { return new ClearIndicesCacheRequestBuilder(this).setIndices(indices); } + @Override + public ActionFuture create(final CreateIndexRequest request) { + return execute(CreateIndexAction.INSTANCE, request); + } + + @Override + public void create(final CreateIndexRequest request, final ActionListener listener) { + execute(CreateIndexAction.INSTANCE, request, listener); + } + @Override public CreateIndexRequestBuilder prepareCreate(String index) { return new CreateIndexRequestBuilder(this, index); } + @Override + public ActionFuture delete(final DeleteIndexRequest request) { + return execute(DeleteIndexAction.INSTANCE, request); + } + + @Override + public void delete(final DeleteIndexRequest request, final ActionListener listener) { + execute(DeleteIndexAction.INSTANCE, request, listener); + } + @Override public DeleteIndexRequestBuilder prepareDelete(String... indices) { return new DeleteIndexRequestBuilder(this, indices); } + @Override + public ActionFuture close(final CloseIndexRequest request) { + return execute(CloseIndexAction.INSTANCE, request); + } + + @Override + public void close(final CloseIndexRequest request, final ActionListener listener) { + execute(CloseIndexAction.INSTANCE, request, listener); + } + @Override public CloseIndexRequestBuilder prepareClose(String index) { return new CloseIndexRequestBuilder(this, index); } + @Override + public ActionFuture open(final OpenIndexRequest request) { + return execute(OpenIndexAction.INSTANCE, request); + } + + @Override + public void open(final OpenIndexRequest request, final ActionListener listener) { + execute(OpenIndexAction.INSTANCE, request, listener); + } + @Override public OpenIndexRequestBuilder prepareOpen(String index) { return new OpenIndexRequestBuilder(this, index); } + @Override + public ActionFuture flush(final FlushRequest request) { + return execute(FlushAction.INSTANCE, request); + } + + @Override + public void flush(final FlushRequest request, final ActionListener listener) { + execute(FlushAction.INSTANCE, request, listener); + } + @Override public FlushRequestBuilder prepareFlush(String... indices) { return new FlushRequestBuilder(this).setIndices(indices); } + @Override + public ActionFuture gatewaySnapshot(final GatewaySnapshotRequest request) { + return execute(GatewaySnapshotAction.INSTANCE, request); + } + + @Override + public void gatewaySnapshot(final GatewaySnapshotRequest request, final ActionListener listener) { + execute(GatewaySnapshotAction.INSTANCE, request, listener); + } + @Override public GatewaySnapshotRequestBuilder prepareGatewaySnapshot(String... indices) { return new GatewaySnapshotRequestBuilder(this).setIndices(indices); } + @Override + public ActionFuture putMapping(final PutMappingRequest request) { + return execute(PutMappingAction.INSTANCE, request); + } + + @Override + public void putMapping(final PutMappingRequest request, final ActionListener listener) { + execute(PutMappingAction.INSTANCE, request, listener); + } + @Override public PutMappingRequestBuilder preparePutMapping(String... indices) { return new PutMappingRequestBuilder(this).setIndices(indices); } + @Override + public ActionFuture deleteMapping(final DeleteMappingRequest request) { + return execute(DeleteMappingAction.INSTANCE, request); + } + + @Override + public void deleteMapping(final DeleteMappingRequest request, final ActionListener listener) { + execute(DeleteMappingAction.INSTANCE, request, listener); + } + @Override public DeleteMappingRequestBuilder prepareDeleteMapping(String... indices) { return new DeleteMappingRequestBuilder(this).setIndices(indices); } + @Override + public ActionFuture optimize(final OptimizeRequest request) { + return execute(OptimizeAction.INSTANCE, request); + } + + @Override + public void optimize(final OptimizeRequest request, final ActionListener listener) { + execute(OptimizeAction.INSTANCE, request, listener); + } + @Override public OptimizeRequestBuilder prepareOptimize(String... indices) { return new OptimizeRequestBuilder(this).setIndices(indices); } + @Override + public ActionFuture refresh(final RefreshRequest request) { + return execute(RefreshAction.INSTANCE, request); + } + + @Override + public void refresh(final RefreshRequest request, final ActionListener listener) { + execute(RefreshAction.INSTANCE, request, listener); + } + @Override public RefreshRequestBuilder prepareRefresh(String... indices) { return new RefreshRequestBuilder(this).setIndices(indices); } + @Override + public ActionFuture stats(final IndicesStatsRequest request) { + return execute(IndicesStatsAction.INSTANCE, request); + } + + @Override + public void stats(final IndicesStatsRequest request, final ActionListener listener) { + execute(IndicesStatsAction.INSTANCE, request, listener); + } + @Override public IndicesStatsRequestBuilder prepareStats(String... indices) { return new IndicesStatsRequestBuilder(this).setIndices(indices); } + @Override + public ActionFuture status(final IndicesStatusRequest request) { + return execute(IndicesStatusAction.INSTANCE, request); + } + + @Override + public void status(final IndicesStatusRequest request, final ActionListener listener) { + execute(IndicesStatusAction.INSTANCE, request, listener); + } + @Override public IndicesStatusRequestBuilder prepareStatus(String... indices) { return new IndicesStatusRequestBuilder(this).setIndices(indices); } + @Override + public ActionFuture segments(final IndicesSegmentsRequest request) { + return execute(IndicesSegmentsAction.INSTANCE, request); + } + + @Override + public void segments(final IndicesSegmentsRequest request, final ActionListener listener) { + execute(IndicesSegmentsAction.INSTANCE, request, listener); + } + @Override public IndicesSegmentsRequestBuilder prepareSegments(String... indices) { return new IndicesSegmentsRequestBuilder(this).setIndices(indices); } + @Override + public ActionFuture updateSettings(final UpdateSettingsRequest request) { + return execute(UpdateSettingsAction.INSTANCE, request); + } + + @Override + public void updateSettings(final UpdateSettingsRequest request, final ActionListener listener) { + execute(UpdateSettingsAction.INSTANCE, request, listener); + } + @Override public UpdateSettingsRequestBuilder prepareUpdateSettings(String... indices) { return new UpdateSettingsRequestBuilder(this).setIndices(indices); } + @Override + public ActionFuture analyze(final AnalyzeRequest request) { + return execute(AnalyzeAction.INSTANCE, request); + } + + @Override + public void analyze(final AnalyzeRequest request, final ActionListener listener) { + execute(AnalyzeAction.INSTANCE, request, listener); + } + @Override public AnalyzeRequestBuilder prepareAnalyze(@Nullable String index, String text) { return new AnalyzeRequestBuilder(this, index, text); @@ -143,16 +393,46 @@ public abstract class AbstractIndicesAdminClient implements InternalIndicesAdmin return new AnalyzeRequestBuilder(this, null, text); } + @Override + public ActionFuture putTemplate(final PutIndexTemplateRequest request) { + return execute(PutIndexTemplateAction.INSTANCE, request); + } + + @Override + public void putTemplate(final PutIndexTemplateRequest request, final ActionListener listener) { + execute(PutIndexTemplateAction.INSTANCE, request, listener); + } + @Override public PutIndexTemplateRequestBuilder preparePutTemplate(String name) { return new PutIndexTemplateRequestBuilder(this, name); } + @Override + public ActionFuture deleteTemplate(final DeleteIndexTemplateRequest request) { + return execute(DeleteIndexTemplateAction.INSTANCE, request); + } + + @Override + public void deleteTemplate(final DeleteIndexTemplateRequest request, final ActionListener listener) { + execute(DeleteIndexTemplateAction.INSTANCE, request, listener); + } + @Override public DeleteIndexTemplateRequestBuilder prepareDeleteTemplate(String name) { return new DeleteIndexTemplateRequestBuilder(this, name); } + @Override + public ActionFuture validateQuery(final ValidateQueryRequest request) { + return execute(ValidateQueryAction.INSTANCE, request); + } + + @Override + public void validateQuery(final ValidateQueryRequest request, final ActionListener listener) { + execute(ValidateQueryAction.INSTANCE, request, listener); + } + @Override public ValidateQueryRequestBuilder prepareValidateQuery(String... indices) { return new ValidateQueryRequestBuilder(this).setIndices(indices); diff --git a/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/src/main/java/org/elasticsearch/client/transport/TransportClient.java index c13d0d9ff6c..d11ea6f9c33 100644 --- a/src/main/java/org/elasticsearch/client/transport/TransportClient.java +++ b/src/main/java/org/elasticsearch/client/transport/TransportClient.java @@ -21,8 +21,7 @@ package org.elasticsearch.client.transport; import com.google.common.collect.ImmutableList; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.*; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.count.CountRequest; @@ -47,7 +46,6 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.support.AbstractClient; -import org.elasticsearch.client.transport.action.ClientTransportActionModule; import org.elasticsearch.client.transport.support.InternalTransportClient; import org.elasticsearch.cluster.ClusterNameModule; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -159,7 +157,7 @@ public class TransportClient extends AbstractClient { modules.add(new ThreadPoolModule(settings)); modules.add(new TransportSearchModule()); modules.add(new TransportModule(settings)); - modules.add(new ClientTransportActionModule()); + modules.add(new ActionModule(true)); modules.add(new ClientTransportModule()); injector = modules.createInjector(); @@ -249,6 +247,16 @@ public class TransportClient extends AbstractClient { return internalClient.admin(); } + @Override + public > ActionFuture execute(Action action, Request request) { + return internalClient.execute(action, request); + } + + @Override + public > void execute(Action action, Request request, ActionListener listener) { + internalClient.execute(action, request, listener); + } + @Override public ActionFuture index(IndexRequest request) { return internalClient.index(request); diff --git a/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java index bb1ba08a9e6..c1ef7ddf6ad 100644 --- a/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java +++ b/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java @@ -24,8 +24,8 @@ import com.google.common.collect.Maps; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterName; @@ -267,7 +267,7 @@ public class TransportClientNodesService extends AbstractComponent { } } try { - NodesInfoResponse nodeInfo = transportService.submitRequest(node, TransportActions.Admin.Cluster.Node.INFO, Requests.nodesInfoRequest("_local"), new FutureTransportResponseHandler() { + NodesInfoResponse nodeInfo = transportService.submitRequest(node, NodesInfoAction.NAME, Requests.nodesInfoRequest("_local"), new FutureTransportResponseHandler() { @Override public NodesInfoResponse newInstance() { return new NodesInfoResponse(); @@ -312,7 +312,7 @@ public class TransportClientNodesService extends AbstractComponent { public void run() { try { transportService.connectToNode(listedNode); // make sure we are connected to it - transportService.sendRequest(listedNode, TransportActions.Admin.Cluster.Node.INFO, Requests.nodesInfoRequest("_all"), new BaseTransportResponseHandler() { + transportService.sendRequest(listedNode, NodesInfoAction.NAME, Requests.nodesInfoRequest("_all"), new BaseTransportResponseHandler() { @Override public NodesInfoResponse newInstance() { diff --git a/src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java b/src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java deleted file mode 100644 index 67a84b71438..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/ClientTransportActionModule.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action; - -import com.google.common.collect.Maps; -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.client.transport.action.admin.cluster.health.ClientTransportClusterHealthAction; -import org.elasticsearch.client.transport.action.admin.cluster.node.info.ClientTransportNodesInfoAction; -import org.elasticsearch.client.transport.action.admin.cluster.node.restart.ClientTransportNodesRestartAction; -import org.elasticsearch.client.transport.action.admin.cluster.node.shutdown.ClientTransportNodesShutdownAction; -import org.elasticsearch.client.transport.action.admin.cluster.node.stats.ClientTransportNodesStatsAction; -import org.elasticsearch.client.transport.action.admin.cluster.ping.broadcast.ClientTransportBroadcastPingAction; -import org.elasticsearch.client.transport.action.admin.cluster.ping.replication.ClientTransportReplicationPingAction; -import org.elasticsearch.client.transport.action.admin.cluster.ping.single.ClientTransportSinglePingAction; -import org.elasticsearch.client.transport.action.admin.cluster.reroute.ClientTransportClusterRerouteAction; -import org.elasticsearch.client.transport.action.admin.cluster.settings.ClientTransportClusterUpdateSettingsAction; -import org.elasticsearch.client.transport.action.admin.cluster.state.ClientTransportClusterStateAction; -import org.elasticsearch.client.transport.action.admin.indices.alias.ClientTransportIndicesAliasesAction; -import org.elasticsearch.client.transport.action.admin.indices.analyze.ClientTransportAnalyzeAction; -import org.elasticsearch.client.transport.action.admin.indices.cache.clear.ClientTransportClearIndicesCacheAction; -import org.elasticsearch.client.transport.action.admin.indices.close.ClientTransportCloseIndexAction; -import org.elasticsearch.client.transport.action.admin.indices.create.ClientTransportCreateIndexAction; -import org.elasticsearch.client.transport.action.admin.indices.delete.ClientTransportDeleteIndexAction; -import org.elasticsearch.client.transport.action.admin.indices.exists.ClientTransportIndicesExistsAction; -import org.elasticsearch.client.transport.action.admin.indices.flush.ClientTransportFlushAction; -import org.elasticsearch.client.transport.action.admin.indices.gateway.snapshot.ClientTransportGatewaySnapshotAction; -import org.elasticsearch.client.transport.action.admin.indices.mapping.delete.ClientTransportDeleteMappingAction; -import org.elasticsearch.client.transport.action.admin.indices.mapping.put.ClientTransportPutMappingAction; -import org.elasticsearch.client.transport.action.admin.indices.open.ClientTransportOpenIndexAction; -import org.elasticsearch.client.transport.action.admin.indices.optimize.ClientTransportOptimizeAction; -import org.elasticsearch.client.transport.action.admin.indices.refresh.ClientTransportRefreshAction; -import org.elasticsearch.client.transport.action.admin.indices.segments.ClientTransportIndicesSegmentsAction; -import org.elasticsearch.client.transport.action.admin.indices.settings.ClientTransportUpdateSettingsAction; -import org.elasticsearch.client.transport.action.admin.indices.stats.ClientTransportIndicesStatsAction; -import org.elasticsearch.client.transport.action.admin.indices.status.ClientTransportIndicesStatusAction; -import org.elasticsearch.client.transport.action.admin.indices.template.delete.ClientTransportDeleteIndexTemplateAction; -import org.elasticsearch.client.transport.action.admin.indices.template.put.ClientTransportPutIndexTemplateAction; -import org.elasticsearch.client.transport.action.admin.indices.validate.query.ClientTransportValidateQueryAction; -import org.elasticsearch.client.transport.action.bulk.ClientTransportBulkAction; -import org.elasticsearch.client.transport.action.count.ClientTransportCountAction; -import org.elasticsearch.client.transport.action.delete.ClientTransportDeleteAction; -import org.elasticsearch.client.transport.action.deletebyquery.ClientTransportDeleteByQueryAction; -import org.elasticsearch.client.transport.action.get.ClientTransportGetAction; -import org.elasticsearch.client.transport.action.get.ClientTransportMultiGetAction; -import org.elasticsearch.client.transport.action.index.ClientTransportIndexAction; -import org.elasticsearch.client.transport.action.percolate.ClientTransportPercolateAction; -import org.elasticsearch.client.transport.action.search.ClientTransportSearchAction; -import org.elasticsearch.client.transport.action.search.ClientTransportSearchScrollAction; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.client.transport.action.update.ClientTransportUpdateAction; -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.inject.multibindings.MapBinder; - -import java.util.Map; - -/** - * - */ -public class ClientTransportActionModule extends AbstractModule { - - private final Map actions = Maps.newHashMap(); - - static class ActionEntry { - public final String name; - public final Class action; - public final Class[] supportActions; - - ActionEntry(String name, Class action, Class... supportActions) { - this.name = name; - this.action = action; - this.supportActions = supportActions; - } - } - - /** - * Registers a custom action under the provided action name, the actual action implementation, and - * any supported actions (bind as singletons). - * - * @param actionName The action name - * @param action The action itself - * @param supportActions Support actions. - */ - public void registerAction(String actionName, Class action, Class... supportActions) { - actions.put(actionName, new ActionEntry(actionName, action, supportActions)); - } - - @Override - protected void configure() { - registerAction(TransportActions.INDEX, ClientTransportIndexAction.class); - registerAction(TransportActions.DELETE, ClientTransportDeleteAction.class); - registerAction(TransportActions.DELETE_BY_QUERY, ClientTransportDeleteByQueryAction.class); - registerAction(TransportActions.GET, ClientTransportGetAction.class); - registerAction(TransportActions.MULTI_GET, ClientTransportMultiGetAction.class); - registerAction(TransportActions.COUNT, ClientTransportCountAction.class); - registerAction(TransportActions.SEARCH, ClientTransportSearchAction.class); - registerAction(TransportActions.SEARCH_SCROLL, ClientTransportSearchScrollAction.class); - registerAction(TransportActions.BULK, ClientTransportBulkAction.class); - registerAction(TransportActions.PERCOLATE, ClientTransportPercolateAction.class); - registerAction(TransportActions.UPDATE, ClientTransportUpdateAction.class); - - registerAction(TransportActions.Admin.Indices.EXISTS, ClientTransportIndicesExistsAction.class); - registerAction(TransportActions.Admin.Indices.STATS, ClientTransportIndicesStatsAction.class); - registerAction(TransportActions.Admin.Indices.STATUS, ClientTransportIndicesStatusAction.class); - registerAction(TransportActions.Admin.Indices.SEGMENTS, ClientTransportIndicesSegmentsAction.class); - registerAction(TransportActions.Admin.Indices.REFRESH, ClientTransportRefreshAction.class); - registerAction(TransportActions.Admin.Indices.FLUSH, ClientTransportFlushAction.class); - registerAction(TransportActions.Admin.Indices.OPTIMIZE, ClientTransportOptimizeAction.class); - registerAction(TransportActions.Admin.Indices.CREATE, ClientTransportCreateIndexAction.class); - registerAction(TransportActions.Admin.Indices.DELETE, ClientTransportDeleteIndexAction.class); - registerAction(TransportActions.Admin.Indices.CLOSE, ClientTransportCloseIndexAction.class); - registerAction(TransportActions.Admin.Indices.OPEN, ClientTransportOpenIndexAction.class); - registerAction(TransportActions.Admin.Indices.Mapping.PUT, ClientTransportPutMappingAction.class); - registerAction(TransportActions.Admin.Indices.Mapping.DELETE, ClientTransportDeleteMappingAction.class); - registerAction(TransportActions.Admin.Indices.Gateway.SNAPSHOT, ClientTransportGatewaySnapshotAction.class); - registerAction(TransportActions.Admin.Indices.ALIASES, ClientTransportIndicesAliasesAction.class); - registerAction(TransportActions.Admin.Indices.Cache.CLEAR, ClientTransportClearIndicesCacheAction.class); - registerAction(TransportActions.Admin.Indices.UPDATE_SETTINGS, ClientTransportUpdateSettingsAction.class); - registerAction(TransportActions.Admin.Indices.ANALYZE, ClientTransportAnalyzeAction.class); - registerAction(TransportActions.Admin.Indices.Template.PUT, ClientTransportPutIndexTemplateAction.class); - registerAction(TransportActions.Admin.Indices.Template.DELETE, ClientTransportDeleteIndexTemplateAction.class); - registerAction(TransportActions.Admin.Indices.Validate.QUERY, ClientTransportValidateQueryAction.class); - - registerAction(TransportActions.Admin.Cluster.Node.INFO, ClientTransportNodesInfoAction.class); - registerAction(TransportActions.Admin.Cluster.Node.STATS, ClientTransportNodesStatsAction.class); - registerAction(TransportActions.Admin.Cluster.Node.SHUTDOWN, ClientTransportNodesShutdownAction.class); - registerAction(TransportActions.Admin.Cluster.Node.RESTART, ClientTransportNodesRestartAction.class); - registerAction(TransportActions.Admin.Cluster.Ping.SINGLE, ClientTransportSinglePingAction.class); - registerAction(TransportActions.Admin.Cluster.Ping.REPLICATION, ClientTransportReplicationPingAction.class); - registerAction(TransportActions.Admin.Cluster.Ping.BROADCAST, ClientTransportBroadcastPingAction.class); - registerAction(TransportActions.Admin.Cluster.STATE, ClientTransportClusterStateAction.class); - registerAction(TransportActions.Admin.Cluster.HEALTH, ClientTransportClusterHealthAction.class); - registerAction(TransportActions.Admin.Cluster.UPDATE_SETTINGS, ClientTransportClusterUpdateSettingsAction.class); - registerAction(TransportActions.Admin.Cluster.REROUTE, ClientTransportClusterRerouteAction.class); - - MapBinder actionsBinder - = MapBinder.newMapBinder(binder(), String.class, BaseClientTransportAction.class); - - for (Map.Entry entry : actions.entrySet()) { - actionsBinder.addBinding(entry.getKey()).to(entry.getValue().action).asEagerSingleton(); - for (Class supportAction : entry.getValue().supportActions) { - bind(supportAction).asEagerSingleton(); - } - } - } -} diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/health/ClientTransportClusterHealthAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/health/ClientTransportClusterHealthAction.java deleted file mode 100644 index 003d42294d1..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/health/ClientTransportClusterHealthAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.cluster.health; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportClusterHealthAction extends BaseClientTransportAction { - - @Inject - public ClientTransportClusterHealthAction(Settings settings, TransportService transportService) { - super(settings, transportService, ClusterHealthResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Cluster.HEALTH; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/info/ClientTransportNodesInfoAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/info/ClientTransportNodesInfoAction.java deleted file mode 100644 index 48a9bde1642..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/info/ClientTransportNodesInfoAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.cluster.node.info; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportNodesInfoAction extends BaseClientTransportAction { - - @Inject - public ClientTransportNodesInfoAction(Settings settings, TransportService transportService) { - super(settings, transportService, NodesInfoResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Cluster.Node.INFO; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/restart/ClientTransportNodesRestartAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/restart/ClientTransportNodesRestartAction.java deleted file mode 100644 index 307e5a6777d..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/restart/ClientTransportNodesRestartAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.cluster.node.restart; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartRequest; -import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportNodesRestartAction extends BaseClientTransportAction { - - @Inject - public ClientTransportNodesRestartAction(Settings settings, TransportService transportService) { - super(settings, transportService, NodesRestartResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Cluster.Node.RESTART; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/shutdown/ClientTransportNodesShutdownAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/shutdown/ClientTransportNodesShutdownAction.java deleted file mode 100644 index ee2b4b71863..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/shutdown/ClientTransportNodesShutdownAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.cluster.node.shutdown; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownRequest; -import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportNodesShutdownAction extends BaseClientTransportAction { - - @Inject - public ClientTransportNodesShutdownAction(Settings settings, TransportService transportService) { - super(settings, transportService, NodesShutdownResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Cluster.Node.SHUTDOWN; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/stats/ClientTransportNodesStatsAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/stats/ClientTransportNodesStatsAction.java deleted file mode 100644 index bc404dadd49..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/node/stats/ClientTransportNodesStatsAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.cluster.node.stats; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportNodesStatsAction extends BaseClientTransportAction { - - @Inject - public ClientTransportNodesStatsAction(Settings settings, TransportService transportService) { - super(settings, transportService, NodesStatsResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Cluster.Node.STATS; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/ping/broadcast/ClientTransportBroadcastPingAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/ping/broadcast/ClientTransportBroadcastPingAction.java deleted file mode 100644 index ca91db202bb..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/ping/broadcast/ClientTransportBroadcastPingAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.cluster.ping.broadcast; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingRequest; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportBroadcastPingAction extends BaseClientTransportAction { - - @Inject - public ClientTransportBroadcastPingAction(Settings settings, TransportService transportService) { - super(settings, transportService, BroadcastPingResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Cluster.Ping.BROADCAST; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/ping/replication/ClientTransportReplicationPingAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/ping/replication/ClientTransportReplicationPingAction.java deleted file mode 100644 index 7c1744c10f2..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/ping/replication/ClientTransportReplicationPingAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.cluster.ping.replication; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingRequest; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportReplicationPingAction extends BaseClientTransportAction { - - @Inject - public ClientTransportReplicationPingAction(Settings settings, TransportService transportService) { - super(settings, transportService, ReplicationPingResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Cluster.Ping.REPLICATION; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/ping/single/ClientTransportSinglePingAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/ping/single/ClientTransportSinglePingAction.java deleted file mode 100644 index 2d6043bac00..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/ping/single/ClientTransportSinglePingAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.cluster.ping.single; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingRequest; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportSinglePingAction extends BaseClientTransportAction { - - @Inject - public ClientTransportSinglePingAction(Settings settings, TransportService transportService) { - super(settings, transportService, SinglePingResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Cluster.Ping.SINGLE; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/reroute/ClientTransportClusterRerouteAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/reroute/ClientTransportClusterRerouteAction.java deleted file mode 100644 index 320ac108ded..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/reroute/ClientTransportClusterRerouteAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.cluster.reroute; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest; -import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportClusterRerouteAction extends BaseClientTransportAction { - - @Inject - public ClientTransportClusterRerouteAction(Settings settings, TransportService transportService) { - super(settings, transportService, ClusterRerouteResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Cluster.REROUTE; - } -} diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/settings/ClientTransportClusterUpdateSettingsAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/settings/ClientTransportClusterUpdateSettingsAction.java deleted file mode 100644 index 9e5af794204..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/settings/ClientTransportClusterUpdateSettingsAction.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.cluster.settings; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; -import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - */ -public class ClientTransportClusterUpdateSettingsAction extends BaseClientTransportAction { - - @Inject - public ClientTransportClusterUpdateSettingsAction(Settings settings, TransportService transportService) { - super(settings, transportService, ClusterUpdateSettingsResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Cluster.UPDATE_SETTINGS; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/state/ClientTransportClusterStateAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/state/ClientTransportClusterStateAction.java deleted file mode 100644 index f05a4d0e543..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/cluster/state/ClientTransportClusterStateAction.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.cluster.state; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - */ -public class ClientTransportClusterStateAction extends BaseClientTransportAction { - - @Inject - public ClientTransportClusterStateAction(Settings settings, TransportService transportService) { - super(settings, transportService, ClusterStateResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Cluster.STATE; - } -} diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/alias/ClientTransportIndicesAliasesAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/alias/ClientTransportIndicesAliasesAction.java deleted file mode 100644 index 31431747052..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/alias/ClientTransportIndicesAliasesAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.alias; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportIndicesAliasesAction extends BaseClientTransportAction { - - @Inject - public ClientTransportIndicesAliasesAction(Settings settings, TransportService transportService) { - super(settings, transportService, IndicesAliasesResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.ALIASES; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/analyze/ClientTransportAnalyzeAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/analyze/ClientTransportAnalyzeAction.java deleted file mode 100644 index 64f8dd89dd9..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/analyze/ClientTransportAnalyzeAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.analyze; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportAnalyzeAction extends BaseClientTransportAction { - - @Inject - public ClientTransportAnalyzeAction(Settings settings, TransportService transportService) { - super(settings, transportService, AnalyzeResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.ANALYZE; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/cache/clear/ClientTransportClearIndicesCacheAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/cache/clear/ClientTransportClearIndicesCacheAction.java deleted file mode 100644 index 944736b3db1..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/cache/clear/ClientTransportClearIndicesCacheAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.cache.clear; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportClearIndicesCacheAction extends BaseClientTransportAction { - - @Inject - public ClientTransportClearIndicesCacheAction(Settings settings, TransportService transportService) { - super(settings, transportService, ClearIndicesCacheResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.Cache.CLEAR; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/close/ClientTransportCloseIndexAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/close/ClientTransportCloseIndexAction.java deleted file mode 100644 index 4644c105fed..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/close/ClientTransportCloseIndexAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.close; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; -import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportCloseIndexAction extends BaseClientTransportAction { - - @Inject - public ClientTransportCloseIndexAction(Settings settings, TransportService transportService) { - super(settings, transportService, CloseIndexResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.CLOSE; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/create/ClientTransportCreateIndexAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/create/ClientTransportCreateIndexAction.java deleted file mode 100644 index e6fde7cdbb2..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/create/ClientTransportCreateIndexAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.create; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportCreateIndexAction extends BaseClientTransportAction { - - @Inject - public ClientTransportCreateIndexAction(Settings settings, TransportService transportService) { - super(settings, transportService, CreateIndexResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.CREATE; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/delete/ClientTransportDeleteIndexAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/delete/ClientTransportDeleteIndexAction.java deleted file mode 100644 index 5e3287316a4..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/delete/ClientTransportDeleteIndexAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.delete; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportDeleteIndexAction extends BaseClientTransportAction { - - @Inject - public ClientTransportDeleteIndexAction(Settings settings, TransportService transportService) { - super(settings, transportService, DeleteIndexResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.DELETE; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/exists/ClientTransportIndicesExistsAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/exists/ClientTransportIndicesExistsAction.java deleted file mode 100644 index bbfa60fd74b..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/exists/ClientTransportIndicesExistsAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.exists; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.exists.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.IndicesExistsResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportIndicesExistsAction extends BaseClientTransportAction { - - @Inject - public ClientTransportIndicesExistsAction(Settings settings, TransportService transportService) { - super(settings, transportService, IndicesExistsResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.EXISTS; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/flush/ClientTransportFlushAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/flush/ClientTransportFlushAction.java deleted file mode 100644 index b302f6cf519..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/flush/ClientTransportFlushAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.flush; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.flush.FlushRequest; -import org.elasticsearch.action.admin.indices.flush.FlushResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportFlushAction extends BaseClientTransportAction { - - @Inject - public ClientTransportFlushAction(Settings settings, TransportService transportService) { - super(settings, transportService, FlushResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.FLUSH; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/gateway/snapshot/ClientTransportGatewaySnapshotAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/gateway/snapshot/ClientTransportGatewaySnapshotAction.java deleted file mode 100644 index 1941c46eb61..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/gateway/snapshot/ClientTransportGatewaySnapshotAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.gateway.snapshot; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotRequest; -import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportGatewaySnapshotAction extends BaseClientTransportAction { - - @Inject - public ClientTransportGatewaySnapshotAction(Settings settings, TransportService transportService) { - super(settings, transportService, GatewaySnapshotResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.Gateway.SNAPSHOT; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/mapping/delete/ClientTransportDeleteMappingAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/mapping/delete/ClientTransportDeleteMappingAction.java deleted file mode 100644 index 6044ced3039..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/mapping/delete/ClientTransportDeleteMappingAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.mapping.delete; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportDeleteMappingAction extends BaseClientTransportAction { - - @Inject - public ClientTransportDeleteMappingAction(Settings settings, TransportService transportService) { - super(settings, transportService, DeleteMappingResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.Mapping.DELETE; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/mapping/put/ClientTransportPutMappingAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/mapping/put/ClientTransportPutMappingAction.java deleted file mode 100644 index e2a1402ff3f..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/mapping/put/ClientTransportPutMappingAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.mapping.put; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportPutMappingAction extends BaseClientTransportAction { - - @Inject - public ClientTransportPutMappingAction(Settings settings, TransportService transportService) { - super(settings, transportService, PutMappingResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.Mapping.PUT; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/open/ClientTransportOpenIndexAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/open/ClientTransportOpenIndexAction.java deleted file mode 100644 index 5f99701dc7e..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/open/ClientTransportOpenIndexAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.open; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; -import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportOpenIndexAction extends BaseClientTransportAction { - - @Inject - public ClientTransportOpenIndexAction(Settings settings, TransportService transportService) { - super(settings, transportService, OpenIndexResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.OPEN; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/optimize/ClientTransportOptimizeAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/optimize/ClientTransportOptimizeAction.java deleted file mode 100644 index 6b3c9bcfa6d..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/optimize/ClientTransportOptimizeAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.optimize; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest; -import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportOptimizeAction extends BaseClientTransportAction { - - @Inject - public ClientTransportOptimizeAction(Settings settings, TransportService transportService) { - super(settings, transportService, OptimizeResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.OPTIMIZE; - } -} diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/refresh/ClientTransportRefreshAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/refresh/ClientTransportRefreshAction.java deleted file mode 100644 index dace2ddac3d..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/refresh/ClientTransportRefreshAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.refresh; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportRefreshAction extends BaseClientTransportAction { - - @Inject - public ClientTransportRefreshAction(Settings settings, TransportService transportService) { - super(settings, transportService, RefreshResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.REFRESH; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/segments/ClientTransportIndicesSegmentsAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/segments/ClientTransportIndicesSegmentsAction.java deleted file mode 100644 index 85d0b69eea0..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/segments/ClientTransportIndicesSegmentsAction.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.segments; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; -import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - */ -public class ClientTransportIndicesSegmentsAction extends BaseClientTransportAction { - - @Inject - public ClientTransportIndicesSegmentsAction(Settings settings, TransportService transportService) { - super(settings, transportService, IndicesSegmentResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.SEGMENTS; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/settings/ClientTransportUpdateSettingsAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/settings/ClientTransportUpdateSettingsAction.java deleted file mode 100644 index 7a0acb500a8..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/settings/ClientTransportUpdateSettingsAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.settings; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.settings.UpdateSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.UpdateSettingsResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportUpdateSettingsAction extends BaseClientTransportAction { - - @Inject - public ClientTransportUpdateSettingsAction(Settings settings, TransportService transportService) { - super(settings, transportService, UpdateSettingsResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.UPDATE_SETTINGS; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/stats/ClientTransportIndicesStatsAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/stats/ClientTransportIndicesStatsAction.java deleted file mode 100644 index 0e65a43e1eb..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/stats/ClientTransportIndicesStatsAction.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.stats; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.stats.IndicesStats; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - */ -public class ClientTransportIndicesStatsAction extends BaseClientTransportAction { - - @Inject - public ClientTransportIndicesStatsAction(Settings settings, TransportService transportService) { - super(settings, transportService, IndicesStats.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.STATS; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/status/ClientTransportIndicesStatusAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/status/ClientTransportIndicesStatusAction.java deleted file mode 100644 index 53f213efcb8..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/status/ClientTransportIndicesStatusAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.status; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.status.IndicesStatusRequest; -import org.elasticsearch.action.admin.indices.status.IndicesStatusResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportIndicesStatusAction extends BaseClientTransportAction { - - @Inject - public ClientTransportIndicesStatusAction(Settings settings, TransportService transportService) { - super(settings, transportService, IndicesStatusResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.STATUS; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/delete/ClientTransportDeleteIndexTemplateAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/delete/ClientTransportDeleteIndexTemplateAction.java deleted file mode 100644 index ab91878f9f0..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/delete/ClientTransportDeleteIndexTemplateAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.template.delete; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportDeleteIndexTemplateAction extends BaseClientTransportAction { - - @Inject - public ClientTransportDeleteIndexTemplateAction(Settings settings, TransportService transportService) { - super(settings, transportService, DeleteIndexTemplateResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.Template.DELETE; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/put/ClientTransportPutIndexTemplateAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/put/ClientTransportPutIndexTemplateAction.java deleted file mode 100644 index 38618792553..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/template/put/ClientTransportPutIndexTemplateAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.admin.indices.template.put; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportPutIndexTemplateAction extends BaseClientTransportAction { - - @Inject - public ClientTransportPutIndexTemplateAction(Settings settings, TransportService transportService) { - super(settings, transportService, PutIndexTemplateResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.Template.PUT; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/validate/query/ClientTransportValidateQueryAction.java b/src/main/java/org/elasticsearch/client/transport/action/admin/indices/validate/query/ClientTransportValidateQueryAction.java deleted file mode 100644 index 0612dfcbe87..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/admin/indices/validate/query/ClientTransportValidateQueryAction.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.elasticsearch.client.transport.action.admin.indices.validate.query; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; -import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportValidateQueryAction extends BaseClientTransportAction { - - @Inject - public ClientTransportValidateQueryAction(Settings settings, TransportService transportService) { - super(settings, transportService, ValidateQueryResponse.class); - } - - @Override - protected String action() { - return TransportActions.Admin.Indices.Validate.QUERY; - } -} diff --git a/src/main/java/org/elasticsearch/client/transport/action/count/ClientTransportCountAction.java b/src/main/java/org/elasticsearch/client/transport/action/count/ClientTransportCountAction.java deleted file mode 100644 index 06e8cf1f4a4..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/count/ClientTransportCountAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.count; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.count.CountRequest; -import org.elasticsearch.action.count.CountResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportCountAction extends BaseClientTransportAction { - - @Inject - public ClientTransportCountAction(Settings settings, TransportService transportService) { - super(settings, transportService, CountResponse.class); - } - - @Override - protected String action() { - return TransportActions.COUNT; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/delete/ClientTransportDeleteAction.java b/src/main/java/org/elasticsearch/client/transport/action/delete/ClientTransportDeleteAction.java deleted file mode 100644 index 064b1c3a3e9..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/delete/ClientTransportDeleteAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.delete; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.delete.DeleteResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportDeleteAction extends BaseClientTransportAction { - - @Inject - public ClientTransportDeleteAction(Settings settings, TransportService transportService) { - super(settings, transportService, DeleteResponse.class); - } - - @Override - protected String action() { - return TransportActions.DELETE; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/deletebyquery/ClientTransportDeleteByQueryAction.java b/src/main/java/org/elasticsearch/client/transport/action/deletebyquery/ClientTransportDeleteByQueryAction.java deleted file mode 100644 index 9e32bceb57c..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/deletebyquery/ClientTransportDeleteByQueryAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.deletebyquery; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.deletebyquery.DeleteByQueryRequest; -import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportDeleteByQueryAction extends BaseClientTransportAction { - - @Inject - public ClientTransportDeleteByQueryAction(Settings settings, TransportService transportService) { - super(settings, transportService, DeleteByQueryResponse.class); - } - - @Override - protected String action() { - return TransportActions.DELETE_BY_QUERY; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/get/ClientTransportGetAction.java b/src/main/java/org/elasticsearch/client/transport/action/get/ClientTransportGetAction.java deleted file mode 100644 index bfbcbae854c..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/get/ClientTransportGetAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.get; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportGetAction extends BaseClientTransportAction { - - @Inject - public ClientTransportGetAction(Settings settings, TransportService transportService) { - super(settings, transportService, GetResponse.class); - } - - @Override - protected String action() { - return TransportActions.GET; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/get/ClientTransportMultiGetAction.java b/src/main/java/org/elasticsearch/client/transport/action/get/ClientTransportMultiGetAction.java deleted file mode 100644 index b0777b677b3..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/get/ClientTransportMultiGetAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.get; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.get.MultiGetRequest; -import org.elasticsearch.action.get.MultiGetResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportMultiGetAction extends BaseClientTransportAction { - - @Inject - public ClientTransportMultiGetAction(Settings settings, TransportService transportService) { - super(settings, transportService, MultiGetResponse.class); - } - - @Override - protected String action() { - return TransportActions.MULTI_GET; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/index/ClientTransportIndexAction.java b/src/main/java/org/elasticsearch/client/transport/action/index/ClientTransportIndexAction.java deleted file mode 100644 index 6ac0d67ec50..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/index/ClientTransportIndexAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.index; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportIndexAction extends BaseClientTransportAction { - - @Inject - public ClientTransportIndexAction(Settings settings, TransportService transportService) { - super(settings, transportService, IndexResponse.class); - } - - @Override - protected String action() { - return TransportActions.INDEX; - } -} diff --git a/src/main/java/org/elasticsearch/client/transport/action/mlt/ClientTransportMoreLikeThisAction.java b/src/main/java/org/elasticsearch/client/transport/action/mlt/ClientTransportMoreLikeThisAction.java deleted file mode 100644 index 10d13c86598..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/mlt/ClientTransportMoreLikeThisAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.mlt; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.mlt.MoreLikeThisRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportMoreLikeThisAction extends BaseClientTransportAction { - - @Inject - public ClientTransportMoreLikeThisAction(Settings settings, TransportService transportService) { - super(settings, transportService, SearchResponse.class); - } - - @Override - protected String action() { - return TransportActions.MORE_LIKE_THIS; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/percolate/ClientTransportPercolateAction.java b/src/main/java/org/elasticsearch/client/transport/action/percolate/ClientTransportPercolateAction.java deleted file mode 100644 index 12876f888c2..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/percolate/ClientTransportPercolateAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.percolate; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.percolate.PercolateRequest; -import org.elasticsearch.action.percolate.PercolateResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportPercolateAction extends BaseClientTransportAction { - - @Inject - public ClientTransportPercolateAction(Settings settings, TransportService transportService) { - super(settings, transportService, PercolateResponse.class); - } - - @Override - protected String action() { - return TransportActions.PERCOLATE; - } -} diff --git a/src/main/java/org/elasticsearch/client/transport/action/search/ClientTransportSearchAction.java b/src/main/java/org/elasticsearch/client/transport/action/search/ClientTransportSearchAction.java deleted file mode 100644 index 66488f83c18..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/search/ClientTransportSearchAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.search; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportSearchAction extends BaseClientTransportAction { - - @Inject - public ClientTransportSearchAction(Settings settings, TransportService transportService) { - super(settings, transportService, SearchResponse.class); - } - - @Override - protected String action() { - return TransportActions.SEARCH; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/search/ClientTransportSearchScrollAction.java b/src/main/java/org/elasticsearch/client/transport/action/search/ClientTransportSearchScrollAction.java deleted file mode 100644 index 628ce9710cb..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/search/ClientTransportSearchScrollAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.search; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchScrollRequest; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportSearchScrollAction extends BaseClientTransportAction { - - @Inject - public ClientTransportSearchScrollAction(Settings settings, TransportService transportService) { - super(settings, transportService, SearchResponse.class); - } - - @Override - protected String action() { - return TransportActions.SEARCH_SCROLL; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/client/transport/action/update/ClientTransportUpdateAction.java b/src/main/java/org/elasticsearch/client/transport/action/update/ClientTransportUpdateAction.java deleted file mode 100644 index deec84c62a3..00000000000 --- a/src/main/java/org/elasticsearch/client/transport/action/update/ClientTransportUpdateAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.transport.action.update; - -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportService; - -/** - * - */ -public class ClientTransportUpdateAction extends BaseClientTransportAction { - - @Inject - public ClientTransportUpdateAction(Settings settings, TransportService transportService) { - super(settings, transportService, UpdateResponse.class); - } - - @Override - protected String action() { - return TransportActions.UPDATE; - } -} diff --git a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java index 8bfb59e35f0..74ff56b4c63 100644 --- a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java +++ b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClient.java @@ -19,53 +19,19 @@ package org.elasticsearch.client.transport.support; +import com.google.common.collect.ImmutableMap; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.count.CountRequest; -import org.elasticsearch.action.count.CountResponse; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.delete.DeleteResponse; -import org.elasticsearch.action.deletebyquery.DeleteByQueryRequest; -import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse; -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.get.MultiGetRequest; -import org.elasticsearch.action.get.MultiGetResponse; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.mlt.MoreLikeThisRequest; -import org.elasticsearch.action.percolate.PercolateRequest; -import org.elasticsearch.action.percolate.PercolateResponse; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchScrollRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.action.*; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.internal.InternalClient; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.client.transport.TransportClientNodesService; -import org.elasticsearch.client.transport.action.bulk.ClientTransportBulkAction; -import org.elasticsearch.client.transport.action.count.ClientTransportCountAction; -import org.elasticsearch.client.transport.action.delete.ClientTransportDeleteAction; -import org.elasticsearch.client.transport.action.deletebyquery.ClientTransportDeleteByQueryAction; -import org.elasticsearch.client.transport.action.get.ClientTransportGetAction; -import org.elasticsearch.client.transport.action.get.ClientTransportMultiGetAction; -import org.elasticsearch.client.transport.action.index.ClientTransportIndexAction; -import org.elasticsearch.client.transport.action.mlt.ClientTransportMoreLikeThisAction; -import org.elasticsearch.client.transport.action.percolate.ClientTransportPercolateAction; -import org.elasticsearch.client.transport.action.search.ClientTransportSearchAction; -import org.elasticsearch.client.transport.action.search.ClientTransportSearchScrollAction; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; -import org.elasticsearch.client.transport.action.update.ClientTransportUpdateAction; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; import java.util.Map; @@ -80,50 +46,23 @@ public class InternalTransportClient extends AbstractClient implements InternalC private final InternalTransportAdminClient adminClient; - private final ClientTransportIndexAction indexAction; - - private final ClientTransportUpdateAction updateAction; - - private final ClientTransportDeleteAction deleteAction; - - private final ClientTransportBulkAction bulkAction; - - private final ClientTransportGetAction getAction; - - private final ClientTransportMultiGetAction multiGetAction; - - private final ClientTransportDeleteByQueryAction deleteByQueryAction; - - private final ClientTransportCountAction countAction; - - private final ClientTransportSearchAction searchAction; - - private final ClientTransportSearchScrollAction searchScrollAction; - - private final ClientTransportMoreLikeThisAction moreLikeThisAction; - - private final ClientTransportPercolateAction percolateAction; + private final ImmutableMap actions; @Inject - public InternalTransportClient(Settings settings, ThreadPool threadPool, + public InternalTransportClient(Settings settings, ThreadPool threadPool, TransportService transportService, TransportClientNodesService nodesService, InternalTransportAdminClient adminClient, - Map actions) { + Map actions) { this.threadPool = threadPool; this.nodesService = nodesService; this.adminClient = adminClient; - this.indexAction = (ClientTransportIndexAction) actions.get(TransportActions.INDEX); - this.updateAction = (ClientTransportUpdateAction) actions.get(TransportActions.UPDATE); - this.deleteAction = (ClientTransportDeleteAction) actions.get(TransportActions.DELETE); - this.bulkAction = (ClientTransportBulkAction) actions.get(TransportActions.BULK); - this.getAction = (ClientTransportGetAction) actions.get(TransportActions.GET); - this.multiGetAction = (ClientTransportMultiGetAction) actions.get(TransportActions.MULTI_GET); - this.deleteByQueryAction = (ClientTransportDeleteByQueryAction) actions.get(TransportActions.DELETE_BY_QUERY); - this.countAction = (ClientTransportCountAction) actions.get(TransportActions.COUNT); - this.searchAction = (ClientTransportSearchAction) actions.get(TransportActions.SEARCH); - this.searchScrollAction = (ClientTransportSearchScrollAction) actions.get(TransportActions.SEARCH_SCROLL); - this.moreLikeThisAction = (ClientTransportMoreLikeThisAction) actions.get(TransportActions.MORE_LIKE_THIS); - this.percolateAction = (ClientTransportPercolateAction) actions.get(TransportActions.PERCOLATE); + MapBuilder actionsBuilder = new MapBuilder(); + for (GenericAction action : actions.values()) { + if (action instanceof Action) { + actionsBuilder.put((Action) action, new TransportActionNodeProxy(action, transportService)); + } + } + this.actions = actionsBuilder.immutableMap(); } @Override @@ -142,241 +81,23 @@ public class InternalTransportClient extends AbstractClient implements InternalC } @Override - public ActionFuture index(final IndexRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { + public > ActionFuture execute(final Action action, final Request request) { + final TransportActionNodeProxy proxy = actions.get(action); + return nodesService.execute(new TransportClientNodesService.NodeCallback>() { @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return indexAction.execute(node, request); + public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { + return proxy.execute(node, request); } }); } @Override - public void index(final IndexRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { + public > void execute(final Action action, final Request request, ActionListener listener) { + final TransportActionNodeProxy proxy = actions.get(action); + nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - indexAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture update(final UpdateRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return updateAction.execute(node, request); - } - }); - } - - @Override - public void update(final UpdateRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - updateAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture delete(final DeleteRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return deleteAction.execute(node, request); - } - }); - } - - @Override - public void delete(final DeleteRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - deleteAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture bulk(final BulkRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return bulkAction.execute(node, request); - } - }); - } - - @Override - public void bulk(final BulkRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - bulkAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture deleteByQuery(final DeleteByQueryRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return deleteByQueryAction.execute(node, request); - } - }); - } - - @Override - public void deleteByQuery(final DeleteByQueryRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - deleteByQueryAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture get(final GetRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return getAction.execute(node, request); - } - }); - } - - @Override - public void get(final GetRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - getAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture multiGet(final MultiGetRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return multiGetAction.execute(node, request); - } - }); - } - - @Override - public void multiGet(final MultiGetRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - multiGetAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture count(final CountRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return countAction.execute(node, request); - } - }); - } - - @Override - public void count(final CountRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - countAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture search(final SearchRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return searchAction.execute(node, request); - } - }); - } - - @Override - public void search(final SearchRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - searchAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture searchScroll(final SearchScrollRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return searchScrollAction.execute(node, request); - } - }); - } - - @Override - public void searchScroll(final SearchScrollRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - searchScrollAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture moreLikeThis(final MoreLikeThisRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return moreLikeThisAction.execute(node, request); - } - }); - } - - @Override - public void moreLikeThis(final MoreLikeThisRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - moreLikeThisAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture percolate(final PercolateRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return percolateAction.execute(node, request); - } - }); - } - - @Override - public void percolate(final PercolateRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - percolateAction.execute(node, request, listener); + public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { + proxy.execute(node, request, listener); } }, listener); } diff --git a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClusterAdminClient.java b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClusterAdminClient.java index f5c73322b07..8af45b1b6c7 100644 --- a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClusterAdminClient.java +++ b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportClusterAdminClient.java @@ -19,51 +19,19 @@ package org.elasticsearch.client.transport.support; +import com.google.common.collect.ImmutableMap; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartRequest; -import org.elasticsearch.action.admin.cluster.node.restart.NodesRestartResponse; -import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownRequest; -import org.elasticsearch.action.admin.cluster.node.shutdown.NodesShutdownResponse; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingRequest; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingResponse; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingRequest; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingResponse; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingRequest; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingResponse; -import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest; -import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; -import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; -import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.*; +import org.elasticsearch.action.admin.cluster.ClusterAction; import org.elasticsearch.client.internal.InternalClusterAdminClient; import org.elasticsearch.client.support.AbstractClusterAdminClient; import org.elasticsearch.client.transport.TransportClientNodesService; -import org.elasticsearch.client.transport.action.admin.cluster.health.ClientTransportClusterHealthAction; -import org.elasticsearch.client.transport.action.admin.cluster.node.info.ClientTransportNodesInfoAction; -import org.elasticsearch.client.transport.action.admin.cluster.node.restart.ClientTransportNodesRestartAction; -import org.elasticsearch.client.transport.action.admin.cluster.node.shutdown.ClientTransportNodesShutdownAction; -import org.elasticsearch.client.transport.action.admin.cluster.node.stats.ClientTransportNodesStatsAction; -import org.elasticsearch.client.transport.action.admin.cluster.ping.broadcast.ClientTransportBroadcastPingAction; -import org.elasticsearch.client.transport.action.admin.cluster.ping.replication.ClientTransportReplicationPingAction; -import org.elasticsearch.client.transport.action.admin.cluster.ping.single.ClientTransportSinglePingAction; -import org.elasticsearch.client.transport.action.admin.cluster.reroute.ClientTransportClusterRerouteAction; -import org.elasticsearch.client.transport.action.admin.cluster.settings.ClientTransportClusterUpdateSettingsAction; -import org.elasticsearch.client.transport.action.admin.cluster.state.ClientTransportClusterStateAction; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; import java.util.Map; @@ -76,44 +44,20 @@ public class InternalTransportClusterAdminClient extends AbstractClusterAdminCli private final ThreadPool threadPool; - private final ClientTransportClusterHealthAction clusterHealthAction; - - private final ClientTransportClusterStateAction clusterStateAction; - - private final ClientTransportClusterRerouteAction clusterRerouteAction; - - private final ClientTransportClusterUpdateSettingsAction clusterUpdateSettingsAction; - - private final ClientTransportSinglePingAction singlePingAction; - - private final ClientTransportReplicationPingAction replicationPingAction; - - private final ClientTransportBroadcastPingAction broadcastPingAction; - - private final ClientTransportNodesInfoAction nodesInfoAction; - - private final ClientTransportNodesStatsAction nodesStatsAction; - - private final ClientTransportNodesShutdownAction nodesShutdownAction; - - private final ClientTransportNodesRestartAction nodesRestartAction; + private final ImmutableMap actions; @Inject - public InternalTransportClusterAdminClient(Settings settings, TransportClientNodesService nodesService, ThreadPool threadPool, - Map actions) { + public InternalTransportClusterAdminClient(Settings settings, TransportClientNodesService nodesService, ThreadPool threadPool, TransportService transportService, + Map actions) { this.nodesService = nodesService; this.threadPool = threadPool; - this.clusterHealthAction = (ClientTransportClusterHealthAction) actions.get(TransportActions.Admin.Cluster.HEALTH); - this.clusterRerouteAction = (ClientTransportClusterRerouteAction) actions.get(TransportActions.Admin.Cluster.REROUTE); - this.clusterStateAction = (ClientTransportClusterStateAction) actions.get(TransportActions.Admin.Cluster.STATE); - this.clusterUpdateSettingsAction = (ClientTransportClusterUpdateSettingsAction) actions.get(TransportActions.Admin.Cluster.UPDATE_SETTINGS); - this.nodesInfoAction = (ClientTransportNodesInfoAction) actions.get(TransportActions.Admin.Cluster.Node.INFO); - this.nodesShutdownAction = (ClientTransportNodesShutdownAction) actions.get(TransportActions.Admin.Cluster.Node.SHUTDOWN); - this.nodesRestartAction = (ClientTransportNodesRestartAction) actions.get(TransportActions.Admin.Cluster.Node.RESTART); - this.singlePingAction = (ClientTransportSinglePingAction) actions.get(TransportActions.Admin.Cluster.Ping.SINGLE); - this.replicationPingAction = (ClientTransportReplicationPingAction) actions.get(TransportActions.Admin.Cluster.Ping.REPLICATION); - this.broadcastPingAction = (ClientTransportBroadcastPingAction) actions.get(TransportActions.Admin.Cluster.Ping.BROADCAST); - this.nodesStatsAction = (ClientTransportNodesStatsAction) actions.get(TransportActions.Admin.Cluster.Node.STATS); + MapBuilder actionsBuilder = new MapBuilder(); + for (GenericAction action : actions.values()) { + if (action instanceof ClusterAction) { + actionsBuilder.put((ClusterAction) action, new TransportActionNodeProxy(action, transportService)); + } + } + this.actions = actionsBuilder.immutableMap(); } @Override @@ -122,221 +66,23 @@ public class InternalTransportClusterAdminClient extends AbstractClusterAdminCli } @Override - public ActionFuture health(final ClusterHealthRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { + public > ActionFuture execute(final ClusterAction action, final Request request) { + final TransportActionNodeProxy proxy = actions.get(action); + return nodesService.execute(new TransportClientNodesService.NodeCallback>() { @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return clusterHealthAction.execute(node, request); + public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { + return proxy.execute(node, request); } }); } @Override - public void health(final ClusterHealthRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { + public > void execute(final ClusterAction action, final Request request, final ActionListener listener) { + final TransportActionNodeProxy proxy = actions.get(action); + nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - clusterHealthAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture state(final ClusterStateRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return clusterStateAction.execute(node, request); - } - }); - } - - @Override - public void state(final ClusterStateRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - clusterStateAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture reroute(final ClusterRerouteRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return clusterRerouteAction.execute(node, request); - } - }); - } - - @Override - public void reroute(final ClusterRerouteRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - clusterRerouteAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture updateSettings(final ClusterUpdateSettingsRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return clusterUpdateSettingsAction.execute(node, request); - } - }); - } - - @Override - public void updateSettings(final ClusterUpdateSettingsRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - clusterUpdateSettingsAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture ping(final SinglePingRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return singlePingAction.execute(node, request); - } - }); - } - - @Override - public void ping(final SinglePingRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - singlePingAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture ping(final BroadcastPingRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return broadcastPingAction.execute(node, request); - } - }); - } - - @Override - public void ping(final BroadcastPingRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - broadcastPingAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture ping(final ReplicationPingRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return replicationPingAction.execute(node, request); - } - }); - } - - @Override - public void ping(final ReplicationPingRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - replicationPingAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture nodesInfo(final NodesInfoRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return nodesInfoAction.execute(node, request); - } - }); - } - - @Override - public void nodesInfo(final NodesInfoRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - nodesInfoAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture nodesStats(final NodesStatsRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return nodesStatsAction.execute(node, request); - } - }); - } - - @Override - public void nodesStats(final NodesStatsRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - nodesStatsAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture nodesShutdown(final NodesShutdownRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return nodesShutdownAction.execute(node, request); - } - }); - } - - @Override - public void nodesShutdown(final NodesShutdownRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - nodesShutdownAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture nodesRestart(final NodesRestartRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return nodesRestartAction.execute(node, request); - } - }); - } - - @Override - public void nodesRestart(final NodesRestartRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - nodesRestartAction.execute(node, request, listener); + public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { + proxy.execute(node, request, listener); } }, listener); } diff --git a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportIndicesAdminClient.java b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportIndicesAdminClient.java index 4f6e5216c50..890ebda775c 100644 --- a/src/main/java/org/elasticsearch/client/transport/support/InternalTransportIndicesAdminClient.java +++ b/src/main/java/org/elasticsearch/client/transport/support/InternalTransportIndicesAdminClient.java @@ -19,81 +19,19 @@ package org.elasticsearch.client.transport.support; +import com.google.common.collect.ImmutableMap; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.TransportActions; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; -import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; -import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; -import org.elasticsearch.action.admin.indices.exists.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.IndicesExistsResponse; -import org.elasticsearch.action.admin.indices.flush.FlushRequest; -import org.elasticsearch.action.admin.indices.flush.FlushResponse; -import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotRequest; -import org.elasticsearch.action.admin.indices.gateway.snapshot.GatewaySnapshotResponse; -import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingResponse; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; -import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; -import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; -import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest; -import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; -import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest; -import org.elasticsearch.action.admin.indices.settings.UpdateSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.UpdateSettingsResponse; -import org.elasticsearch.action.admin.indices.stats.IndicesStats; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; -import org.elasticsearch.action.admin.indices.status.IndicesStatusRequest; -import org.elasticsearch.action.admin.indices.status.IndicesStatusResponse; -import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; -import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; -import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; +import org.elasticsearch.action.*; +import org.elasticsearch.action.admin.indices.IndicesAction; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.client.support.AbstractIndicesAdminClient; import org.elasticsearch.client.transport.TransportClientNodesService; -import org.elasticsearch.client.transport.action.admin.indices.alias.ClientTransportIndicesAliasesAction; -import org.elasticsearch.client.transport.action.admin.indices.analyze.ClientTransportAnalyzeAction; -import org.elasticsearch.client.transport.action.admin.indices.cache.clear.ClientTransportClearIndicesCacheAction; -import org.elasticsearch.client.transport.action.admin.indices.close.ClientTransportCloseIndexAction; -import org.elasticsearch.client.transport.action.admin.indices.create.ClientTransportCreateIndexAction; -import org.elasticsearch.client.transport.action.admin.indices.delete.ClientTransportDeleteIndexAction; -import org.elasticsearch.client.transport.action.admin.indices.exists.ClientTransportIndicesExistsAction; -import org.elasticsearch.client.transport.action.admin.indices.flush.ClientTransportFlushAction; -import org.elasticsearch.client.transport.action.admin.indices.gateway.snapshot.ClientTransportGatewaySnapshotAction; -import org.elasticsearch.client.transport.action.admin.indices.mapping.delete.ClientTransportDeleteMappingAction; -import org.elasticsearch.client.transport.action.admin.indices.mapping.put.ClientTransportPutMappingAction; -import org.elasticsearch.client.transport.action.admin.indices.open.ClientTransportOpenIndexAction; -import org.elasticsearch.client.transport.action.admin.indices.optimize.ClientTransportOptimizeAction; -import org.elasticsearch.client.transport.action.admin.indices.refresh.ClientTransportRefreshAction; -import org.elasticsearch.client.transport.action.admin.indices.segments.ClientTransportIndicesSegmentsAction; -import org.elasticsearch.client.transport.action.admin.indices.settings.ClientTransportUpdateSettingsAction; -import org.elasticsearch.client.transport.action.admin.indices.stats.ClientTransportIndicesStatsAction; -import org.elasticsearch.client.transport.action.admin.indices.status.ClientTransportIndicesStatusAction; -import org.elasticsearch.client.transport.action.admin.indices.template.delete.ClientTransportDeleteIndexTemplateAction; -import org.elasticsearch.client.transport.action.admin.indices.template.put.ClientTransportPutIndexTemplateAction; -import org.elasticsearch.client.transport.action.admin.indices.validate.query.ClientTransportValidateQueryAction; -import org.elasticsearch.client.transport.action.support.BaseClientTransportAction; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; import java.util.Map; @@ -106,74 +44,20 @@ public class InternalTransportIndicesAdminClient extends AbstractIndicesAdminCli private final ThreadPool threadPool; - private final ClientTransportIndicesExistsAction indicesExistsAction; - - private final ClientTransportIndicesStatsAction indicesStatsAction; - - private final ClientTransportIndicesStatusAction indicesStatusAction; - - private final ClientTransportIndicesSegmentsAction indicesSegmentsAction; - - private final ClientTransportCreateIndexAction createIndexAction; - - private final ClientTransportDeleteIndexAction deleteIndexAction; - - private final ClientTransportCloseIndexAction closeIndexAction; - - private final ClientTransportOpenIndexAction openIndexAction; - - private final ClientTransportRefreshAction refreshAction; - - private final ClientTransportFlushAction flushAction; - - private final ClientTransportOptimizeAction optimizeAction; - - private final ClientTransportPutMappingAction putMappingAction; - - private final ClientTransportDeleteMappingAction deleteMappingAction; - - private final ClientTransportGatewaySnapshotAction gatewaySnapshotAction; - - private final ClientTransportIndicesAliasesAction indicesAliasesAction; - - private final ClientTransportClearIndicesCacheAction clearIndicesCacheAction; - - private final ClientTransportUpdateSettingsAction updateSettingsAction; - - private final ClientTransportAnalyzeAction analyzeAction; - - private final ClientTransportPutIndexTemplateAction putIndexTemplateAction; - - private final ClientTransportDeleteIndexTemplateAction deleteIndexTemplateAction; - - private final ClientTransportValidateQueryAction validateQueryAction; + private final ImmutableMap actions; @Inject - public InternalTransportIndicesAdminClient(Settings settings, TransportClientNodesService nodesService, ThreadPool threadPool, - Map actions) { + public InternalTransportIndicesAdminClient(Settings settings, TransportClientNodesService nodesService, TransportService transportService, ThreadPool threadPool, + Map actions) { this.nodesService = nodesService; this.threadPool = threadPool; - this.indicesExistsAction = (ClientTransportIndicesExistsAction) actions.get(TransportActions.Admin.Indices.EXISTS); - this.indicesStatsAction = (ClientTransportIndicesStatsAction) actions.get(TransportActions.Admin.Indices.STATS); - this.indicesStatusAction = (ClientTransportIndicesStatusAction) actions.get(TransportActions.Admin.Indices.STATUS); - this.indicesSegmentsAction = (ClientTransportIndicesSegmentsAction) actions.get(TransportActions.Admin.Indices.SEGMENTS); - this.createIndexAction = (ClientTransportCreateIndexAction) actions.get(TransportActions.Admin.Indices.CREATE); - this.deleteIndexAction = (ClientTransportDeleteIndexAction) actions.get(TransportActions.Admin.Indices.DELETE); - this.closeIndexAction = (ClientTransportCloseIndexAction) actions.get(TransportActions.Admin.Indices.CLOSE); - this.openIndexAction = (ClientTransportOpenIndexAction) actions.get(TransportActions.Admin.Indices.OPEN); - this.refreshAction = (ClientTransportRefreshAction) actions.get(TransportActions.Admin.Indices.REFRESH); - this.flushAction = (ClientTransportFlushAction) actions.get(TransportActions.Admin.Indices.FLUSH); - this.optimizeAction = (ClientTransportOptimizeAction) actions.get(TransportActions.Admin.Indices.OPTIMIZE); - this.putMappingAction = (ClientTransportPutMappingAction) actions.get(TransportActions.Admin.Indices.Mapping.PUT); - this.deleteMappingAction = (ClientTransportDeleteMappingAction) actions.get(TransportActions.Admin.Indices.Mapping.DELETE); - this.gatewaySnapshotAction = (ClientTransportGatewaySnapshotAction) actions.get(TransportActions.Admin.Indices.Gateway.SNAPSHOT); - this.indicesAliasesAction = (ClientTransportIndicesAliasesAction) actions.get(TransportActions.Admin.Indices.ALIASES); - this.clearIndicesCacheAction = (ClientTransportClearIndicesCacheAction) actions.get(TransportActions.Admin.Indices.Cache.CLEAR); - this.updateSettingsAction = (ClientTransportUpdateSettingsAction) actions.get(TransportActions.Admin.Indices.UPDATE_SETTINGS); - this.analyzeAction = (ClientTransportAnalyzeAction) actions.get(TransportActions.Admin.Indices.ANALYZE); - this.putIndexTemplateAction = (ClientTransportPutIndexTemplateAction) actions.get(TransportActions.Admin.Indices.Template.PUT); - this.deleteIndexTemplateAction = (ClientTransportDeleteIndexTemplateAction) actions.get(TransportActions.Admin.Indices.Template.DELETE); - this.validateQueryAction = (ClientTransportValidateQueryAction) actions.get(TransportActions.Admin.Indices.Validate.QUERY); + MapBuilder actionsBuilder = new MapBuilder(); + for (GenericAction action : actions.values()) { + if (action instanceof IndicesAction) { + actionsBuilder.put((IndicesAction) action, new TransportActionNodeProxy(action, transportService)); + } + } + this.actions = actionsBuilder.immutableMap(); } @Override @@ -182,421 +66,23 @@ public class InternalTransportIndicesAdminClient extends AbstractIndicesAdminCli } @Override - public ActionFuture exists(final IndicesExistsRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { + public > ActionFuture execute(final IndicesAction action, final Request request) { + final TransportActionNodeProxy proxy = actions.get(action); + return nodesService.execute(new TransportClientNodesService.NodeCallback>() { @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return indicesExistsAction.execute(node, request); + public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { + return proxy.execute(node, request); } }); } @Override - public void exists(final IndicesExistsRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { + public > void execute(final IndicesAction action, final Request request, ActionListener listener) { + final TransportActionNodeProxy proxy = actions.get(action); + nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - indicesExistsAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture stats(final IndicesStatsRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return indicesStatsAction.execute(node, request); - } - }); - } - - @Override - public void stats(final IndicesStatsRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - indicesStatsAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture status(final IndicesStatusRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return indicesStatusAction.execute(node, request); - } - }); - } - - @Override - public void status(final IndicesStatusRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - indicesStatusAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture segments(final IndicesSegmentsRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return indicesSegmentsAction.execute(node, request); - } - }); - } - - @Override - public void segments(final IndicesSegmentsRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - indicesSegmentsAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture create(final CreateIndexRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return createIndexAction.execute(node, request); - } - }); - } - - @Override - public void create(final CreateIndexRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - createIndexAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture delete(final DeleteIndexRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return deleteIndexAction.execute(node, request); - } - }); - } - - @Override - public void delete(final DeleteIndexRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - deleteIndexAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture close(final CloseIndexRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return closeIndexAction.execute(node, request); - } - }); - } - - @Override - public void close(final CloseIndexRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - closeIndexAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture open(final OpenIndexRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return openIndexAction.execute(node, request); - } - }); - } - - @Override - public void open(final OpenIndexRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - openIndexAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture refresh(final RefreshRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return refreshAction.execute(node, request); - } - }); - } - - @Override - public void refresh(final RefreshRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - refreshAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture flush(final FlushRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return flushAction.execute(node, request); - } - }); - } - - @Override - public void flush(final FlushRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - flushAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture optimize(final OptimizeRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return optimizeAction.execute(node, request); - } - }); - } - - @Override - public void optimize(final OptimizeRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - optimizeAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture putMapping(final PutMappingRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return putMappingAction.execute(node, request); - } - }); - } - - @Override - public void putMapping(final PutMappingRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - putMappingAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture deleteMapping(final DeleteMappingRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return deleteMappingAction.execute(node, request); - } - }); - } - - @Override - public void deleteMapping(final DeleteMappingRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - deleteMappingAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture gatewaySnapshot(final GatewaySnapshotRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return gatewaySnapshotAction.execute(node, request); - } - }); - } - - @Override - public void gatewaySnapshot(final GatewaySnapshotRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - gatewaySnapshotAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture aliases(final IndicesAliasesRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return indicesAliasesAction.execute(node, request); - } - }); - } - - @Override - public void aliases(final IndicesAliasesRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - indicesAliasesAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture clearCache(final ClearIndicesCacheRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return clearIndicesCacheAction.execute(node, request); - } - }); - } - - @Override - public void clearCache(final ClearIndicesCacheRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - clearIndicesCacheAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture updateSettings(final UpdateSettingsRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return updateSettingsAction.execute(node, request); - } - }); - } - - @Override - public void updateSettings(final UpdateSettingsRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - updateSettingsAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture analyze(final AnalyzeRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return analyzeAction.execute(node, request); - } - }); - } - - @Override - public void analyze(final AnalyzeRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - analyzeAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture putTemplate(final PutIndexTemplateRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return putIndexTemplateAction.execute(node, request); - } - }); - } - - @Override - public void putTemplate(final PutIndexTemplateRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - putIndexTemplateAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture deleteTemplate(final DeleteIndexTemplateRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return deleteIndexTemplateAction.execute(node, request); - } - }); - } - - @Override - public void deleteTemplate(final DeleteIndexTemplateRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - deleteIndexTemplateAction.execute(node, request, listener); - } - }, listener); - } - - @Override - public ActionFuture validateQuery(final ValidateQueryRequest request) { - return nodesService.execute(new TransportClientNodesService.NodeCallback>() { - @Override - public ActionFuture doWithNode(DiscoveryNode node) throws ElasticSearchException { - return validateQueryAction.execute(node, request); - } - }); - } - - @Override - public void validateQuery(final ValidateQueryRequest request, final ActionListener listener) { - nodesService.execute(new TransportClientNodesService.NodeListenerCallback() { - @Override - public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { - validateQueryAction.execute(node, request, listener); + public void doWithNode(DiscoveryNode node, ActionListener listener) throws ElasticSearchException { + proxy.execute(node, request, listener); } }, listener); } diff --git a/src/main/java/org/elasticsearch/node/internal/InternalNode.java b/src/main/java/org/elasticsearch/node/internal/InternalNode.java index 99f33e61c6a..a98f54510b6 100644 --- a/src/main/java/org/elasticsearch/node/internal/InternalNode.java +++ b/src/main/java/org/elasticsearch/node/internal/InternalNode.java @@ -21,7 +21,7 @@ package org.elasticsearch.node.internal; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.Version; -import org.elasticsearch.action.TransportActionModule; +import org.elasticsearch.action.ActionModule; import org.elasticsearch.cache.NodeCache; import org.elasticsearch.cache.NodeCacheModule; import org.elasticsearch.client.Client; @@ -141,7 +141,7 @@ public final class InternalNode implements Node { modules.add(new RiversModule(settings)); modules.add(new IndicesModule(settings)); modules.add(new SearchModule()); - modules.add(new TransportActionModule()); + modules.add(new ActionModule(false)); modules.add(new MonitorModule(settings)); modules.add(new GatewayModule(settings)); modules.add(new NodeClientModule()); diff --git a/src/main/java/org/elasticsearch/rest/action/RestActionModule.java b/src/main/java/org/elasticsearch/rest/action/RestActionModule.java index c90fadf63cc..ed9e6905577 100644 --- a/src/main/java/org/elasticsearch/rest/action/RestActionModule.java +++ b/src/main/java/org/elasticsearch/rest/action/RestActionModule.java @@ -27,9 +27,6 @@ import org.elasticsearch.rest.action.admin.cluster.node.info.RestNodesInfoAction import org.elasticsearch.rest.action.admin.cluster.node.restart.RestNodesRestartAction; import org.elasticsearch.rest.action.admin.cluster.node.shutdown.RestNodesShutdownAction; import org.elasticsearch.rest.action.admin.cluster.node.stats.RestNodesStatsAction; -import org.elasticsearch.rest.action.admin.cluster.ping.broadcast.RestBroadcastPingAction; -import org.elasticsearch.rest.action.admin.cluster.ping.replication.RestReplicationPingAction; -import org.elasticsearch.rest.action.admin.cluster.ping.single.RestSinglePingAction; import org.elasticsearch.rest.action.admin.cluster.reroute.RestClusterRerouteAction; import org.elasticsearch.rest.action.admin.cluster.settings.RestClusterGetSettingsAction; import org.elasticsearch.rest.action.admin.cluster.settings.RestClusterUpdateSettingsAction; @@ -103,10 +100,6 @@ public class RestActionModule extends AbstractModule { bind(RestClusterGetSettingsAction.class).asEagerSingleton(); bind(RestClusterRerouteAction.class).asEagerSingleton(); - bind(RestSinglePingAction.class).asEagerSingleton(); - bind(RestBroadcastPingAction.class).asEagerSingleton(); - bind(RestReplicationPingAction.class).asEagerSingleton(); - bind(RestIndicesExistsAction.class).asEagerSingleton(); bind(RestIndicesStatsAction.class).asEagerSingleton(); bind(RestIndicesStatusAction.class).asEagerSingleton(); diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/ping/broadcast/RestBroadcastPingAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/ping/broadcast/RestBroadcastPingAction.java deleted file mode 100644 index 5c66db54594..00000000000 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/ping/broadcast/RestBroadcastPingAction.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.rest.action.admin.cluster.ping.broadcast; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingRequest; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingResponse; -import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.rest.*; -import org.elasticsearch.rest.action.support.RestActions; -import org.elasticsearch.rest.action.support.RestXContentBuilder; - -import java.io.IOException; - -import static org.elasticsearch.rest.RestStatus.OK; -import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastShardsHeader; - -/** - * - */ -public class RestBroadcastPingAction extends BaseRestHandler { - - @Inject - public RestBroadcastPingAction(Settings settings, Client client, RestController controller) { - super(settings, client); - controller.registerHandler(RestRequest.Method.GET, "/{index}/_ping/broadcast", this); - controller.registerHandler(RestRequest.Method.GET, "/_cluster/{index}/_ping/broadcast", this); - } - - @Override - public void handleRequest(final RestRequest request, final RestChannel channel) { - BroadcastPingRequest broadcastPingRequest = new BroadcastPingRequest(RestActions.splitIndices(request.param("index"))); - broadcastPingRequest.queryHint(request.param("query_hint")); - BroadcastOperationThreading operationThreading = BroadcastOperationThreading.fromString(request.param("operation_threading"), BroadcastOperationThreading.SINGLE_THREAD); - if (operationThreading == BroadcastOperationThreading.NO_THREADS) { - // since we don't spawn, don't allow no_threads, but change it to a single thread - operationThreading = BroadcastOperationThreading.SINGLE_THREAD; - } - broadcastPingRequest.operationThreading(operationThreading); - client.admin().cluster().ping(broadcastPingRequest, new ActionListener() { - @Override - public void onResponse(BroadcastPingResponse response) { - try { - XContentBuilder builder = RestXContentBuilder.restContentBuilder(request); - builder.startObject(); - builder.field("ok", true); - buildBroadcastShardsHeader(builder, response); - builder.endObject(); - channel.sendResponse(new XContentRestResponse(request, OK, builder)); - } catch (Exception e) { - onFailure(e); - } - } - - @Override - public void onFailure(Throwable e) { - try { - channel.sendResponse(new XContentThrowableRestResponse(request, e)); - } catch (IOException e1) { - logger.error("Failed to send failure response", e1); - } - } - }); - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/ping/replication/RestReplicationPingAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/ping/replication/RestReplicationPingAction.java deleted file mode 100644 index b92be615217..00000000000 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/ping/replication/RestReplicationPingAction.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.rest.action.admin.cluster.ping.replication; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.ping.replication.IndexReplicationPingResponse; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingRequest; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingResponse; -import org.elasticsearch.action.admin.cluster.ping.replication.ShardReplicationPingRequest; -import org.elasticsearch.action.support.replication.ReplicationType; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.rest.*; -import org.elasticsearch.rest.action.support.RestActions; -import org.elasticsearch.rest.action.support.RestXContentBuilder; - -import java.io.IOException; - -import static org.elasticsearch.rest.RestStatus.OK; - -/** - * - */ -public class RestReplicationPingAction extends BaseRestHandler { - - @Inject - public RestReplicationPingAction(Settings settings, Client client, RestController controller) { - super(settings, client); - controller.registerHandler(RestRequest.Method.GET, "/{index}/_ping/replication", this); - controller.registerHandler(RestRequest.Method.GET, "/_cluster/{index}/_ping/replication", this); - } - - @Override - public void handleRequest(final RestRequest request, final RestChannel channel) { - ReplicationPingRequest replicationPingRequest = new ReplicationPingRequest(RestActions.splitIndices(request.param("index"))); - replicationPingRequest.timeout(request.paramAsTime("timeout", ShardReplicationPingRequest.DEFAULT_TIMEOUT)); - replicationPingRequest.listenerThreaded(false); - String replicationType = request.param("replication"); - if (replicationType != null) { - replicationPingRequest.replicationType(ReplicationType.fromString(replicationType)); - } - client.admin().cluster().ping(replicationPingRequest, new ActionListener() { - @Override - public void onResponse(ReplicationPingResponse result) { - try { - XContentBuilder builder = RestXContentBuilder.restContentBuilder(request); - builder.startObject(); - builder.field("ok", true); - for (IndexReplicationPingResponse indexResponse : result.indices().values()) { - builder.startObject(indexResponse.index()) - .field("ok", true) - .startObject("_shards") - .field("total", indexResponse.totalShards()) - .field("successful", indexResponse.successfulShards()) - .field("failed", indexResponse.failedShards()) - .endObject() - .endObject(); - } - builder.endObject(); - channel.sendResponse(new XContentRestResponse(request, OK, builder)); - } catch (Exception e) { - onFailure(e); - } - } - - @Override - public void onFailure(Throwable e) { - try { - channel.sendResponse(new XContentThrowableRestResponse(request, e)); - } catch (IOException e1) { - logger.error("Failed to send failure response", e1); - } - } - }); - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/ping/single/RestSinglePingAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/ping/single/RestSinglePingAction.java deleted file mode 100644 index fa5d044d8ed..00000000000 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/ping/single/RestSinglePingAction.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.rest.action.admin.cluster.ping.single; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingRequest; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.rest.*; -import org.elasticsearch.rest.action.support.RestXContentBuilder; - -import java.io.IOException; - -import static org.elasticsearch.rest.RestStatus.OK; - -/** - * - */ -public class RestSinglePingAction extends BaseRestHandler { - - @Inject - public RestSinglePingAction(Settings settings, Client client, RestController controller) { - super(settings, client); - controller.registerHandler(RestRequest.Method.GET, "/{index}/{type}/{id}/_ping", this); - controller.registerHandler(RestRequest.Method.GET, "/_cluster/{index}/{type}/{id}/_ping", this); - } - - @Override - public void handleRequest(final RestRequest request, final RestChannel channel) { - SinglePingRequest singlePingRequest = new SinglePingRequest(request.param("index"), request.param("type"), request.param("id")); - // no need to have a threaded listener since we just send back a response - singlePingRequest.listenerThreaded(false); - // if we have a local operation, execute it on a thread since we don't spawn - singlePingRequest.operationThreaded(true); - client.admin().cluster().ping(singlePingRequest, new ActionListener() { - @Override - public void onResponse(SinglePingResponse result) { - try { - XContentBuilder generator = RestXContentBuilder.restContentBuilder(request); - generator.startObject().field("ok", true).endObject(); - channel.sendResponse(new XContentRestResponse(request, OK, generator)); - } catch (Exception e) { - onFailure(e); - } - } - - @Override - public void onFailure(Throwable e) { - try { - channel.sendResponse(new XContentThrowableRestResponse(request, e)); - } catch (IOException e1) { - logger.error("Failed to send failure response", e1); - } - } - }); - } -} \ No newline at end of file diff --git a/src/test/java/org/elasticsearch/test/integration/client/transport/DiscoveryTransportClientTests.java b/src/test/java/org/elasticsearch/test/integration/client/transport/DiscoveryTransportClientTests.java index 202969df818..2f58ed33419 100644 --- a/src/test/java/org/elasticsearch/test/integration/client/transport/DiscoveryTransportClientTests.java +++ b/src/test/java/org/elasticsearch/test/integration/client/transport/DiscoveryTransportClientTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.test.integration.AbstractNodesTests; import org.testng.annotations.AfterMethod; import static org.elasticsearch.client.Requests.createIndexRequest; -import static org.elasticsearch.client.Requests.pingSingleRequest; /** * @@ -55,16 +54,6 @@ public class DiscoveryTransportClientTests extends AbstractNodesTests { Thread.sleep(1000); client.admin().indices().create(createIndexRequest("test")).actionGet(); Thread.sleep(500); - - client.admin().cluster().ping(pingSingleRequest("test").type("person").id("1")).actionGet(); - startNode("server2"); - Thread.sleep(1000); - client.admin().cluster().ping(pingSingleRequest("test").type("person").id("1")).actionGet(); - closeNode("server1"); - Thread.sleep(10000); - client.admin().cluster().ping(pingSingleRequest("test").type("person").id("1")).actionGet(); - closeNode("server2"); - client.admin().cluster().ping(pingSingleRequest("test").type("person").id("1")).actionGet(); } } diff --git a/src/test/java/org/elasticsearch/test/integration/ping/PingActionTests.java b/src/test/java/org/elasticsearch/test/integration/ping/PingActionTests.java deleted file mode 100644 index 3f15ad37209..00000000000 --- a/src/test/java/org/elasticsearch/test/integration/ping/PingActionTests.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.integration.ping; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; -import org.elasticsearch.action.admin.cluster.ping.broadcast.BroadcastPingResponse; -import org.elasticsearch.action.admin.cluster.ping.replication.ReplicationPingResponse; -import org.elasticsearch.action.admin.cluster.ping.single.SinglePingResponse; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.test.integration.AbstractNodesTests; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import static org.elasticsearch.client.Requests.*; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; - -/** - * - */ -public class PingActionTests extends AbstractNodesTests { - - private final ESLogger logger = Loggers.getLogger(PingActionTests.class); - - @BeforeMethod - public void startNodes() { - startNode("server1"); - startNode("server2"); - } - - @AfterMethod - public void closeNodes() { - closeAllNodes(); - } - - @Test - public void testPingActions() throws Exception { - logger.info("Creating index [test1]"); - client("server1").admin().indices().create(createIndexRequest("test1")).actionGet(); - logger.info("Creating index [test2]"); - client("server1").admin().indices().create(createIndexRequest("test2")).actionGet(); - - logger.info("Running Cluster Health"); - ClusterHealthResponse clusterHealth = client("server1").admin().cluster().health(clusterHealthRequest().waitForGreenStatus()).actionGet(); - logger.info("Done Cluster Health, status " + clusterHealth.status()); - assertThat(clusterHealth.timedOut(), equalTo(false)); - assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.GREEN)); - - logger.info("Pinging single person with id 1"); - SinglePingResponse singleResponse = client("server1").admin().cluster().ping(pingSingleRequest("test1").type("person").id("1")).actionGet(); - - logger.info("Broadcast pinging test1 and test2"); - BroadcastPingResponse broadcastResponse = client("server1").admin().cluster().ping(pingBroadcastRequest("test1", "test2")).actionGet(); - assertThat(broadcastResponse.successfulShards(), equalTo(10)); - assertThat(broadcastResponse.failedShards(), equalTo(0)); - - logger.info("Broadcast pinging test1"); - broadcastResponse = client("server1").admin().cluster().ping(pingBroadcastRequest("test1")).actionGet(); - assertThat(broadcastResponse.successfulShards(), equalTo(5)); - assertThat(broadcastResponse.failedShards(), equalTo(0)); - - logger.info("Replication pinging test1 and test2"); - ReplicationPingResponse replicationResponse = client("server1").admin().cluster().ping(pingReplicationRequest("test1", "test2")).actionGet(); - assertThat(replicationResponse.indices().size(), equalTo(2)); - assertThat(replicationResponse.index("test1").successfulShards(), equalTo(5)); - assertThat(replicationResponse.index("test1").failedShards(), equalTo(0)); - } -} From 1d35e27b3b33680bb8395112e6dfa95bdb49e05c Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 15 Jan 2012 16:42:29 +0200 Subject: [PATCH 171/270] Plugins: Allow for plugins to implement onModule method that will be automatically injected with the relevant module type, closes #1613. --- .../elasticsearch/plugins/AbstractPlugin.java | 6 ++- .../org/elasticsearch/plugins/Plugin.java | 10 +++- .../elasticsearch/plugins/PluginsService.java | 51 +++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/plugins/AbstractPlugin.java b/src/main/java/org/elasticsearch/plugins/AbstractPlugin.java index 426d1e4e92f..18cefc387d3 100644 --- a/src/main/java/org/elasticsearch/plugins/AbstractPlugin.java +++ b/src/main/java/org/elasticsearch/plugins/AbstractPlugin.java @@ -30,8 +30,10 @@ import java.util.Collection; /** * A base class for a plugin. - * - * + *

    + * A plugin can be dynamically injected with {@link Module} by implementing onModule(AnyModule) method + * removing the need to override {@link #processModule(org.elasticsearch.common.inject.Module)} and check using + * instanceof. */ public abstract class AbstractPlugin implements Plugin { diff --git a/src/main/java/org/elasticsearch/plugins/Plugin.java b/src/main/java/org/elasticsearch/plugins/Plugin.java index 33852effb11..6fae62a4d8c 100644 --- a/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -28,8 +28,10 @@ import java.util.Collection; /** * An extension point allowing to plug in custom functionality. - * - * + *

    + * A plugin can be dynamically injected with {@link Module} by implementing onModule(AnyModule) method + * removing the need to override {@link #processModule(org.elasticsearch.common.inject.Module)} and check using + * instanceof. */ public interface Plugin { @@ -73,6 +75,10 @@ public interface Plugin { */ Collection> shardServices(); + /** + * Process a specific module. Note, its simpler to implement a custom onModule(AnyModule module) + * method, which will be automatically be called by the relevant type. + */ void processModule(Module module); /** diff --git a/src/main/java/org/elasticsearch/plugins/PluginsService.java b/src/main/java/org/elasticsearch/plugins/PluginsService.java index fe699eabc5f..1eae871796e 100644 --- a/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -24,6 +24,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Inject; @@ -51,6 +52,18 @@ public class PluginsService extends AbstractComponent { private final ImmutableMap plugins; + private final ImmutableMap> onModuleReferences; + + static class OnModuleReference { + public final Class moduleClass; + public final Method onModuleMethod; + + OnModuleReference(Class moduleClass, Method onModuleMethod) { + this.moduleClass = moduleClass; + this.onModuleMethod = onModuleMethod; + } + } + @Inject public PluginsService(Settings settings, Environment environment) { super(settings); @@ -65,6 +78,31 @@ public class PluginsService extends AbstractComponent { logger.info("loaded {}, sites {}", plugins.keySet(), sitePlugins()); this.plugins = ImmutableMap.copyOf(plugins); + + MapBuilder> onModuleReferences = MapBuilder.newMapBuilder(); + for (Plugin plugin : plugins.values()) { + List list = Lists.newArrayList(); + for (Method method : plugin.getClass().getDeclaredMethods()) { + if (!method.getName().equals("onModule")) { + continue; + } + if (method.getParameterTypes().length == 0 || method.getParameterTypes().length > 1) { + logger.warn("Plugin: {} implementing onModule with no parameters or more than one parameter", plugin.name()); + continue; + } + Class moduleClass = method.getParameterTypes()[0]; + if (!Module.class.isAssignableFrom(moduleClass)) { + logger.warn("Plugin: {} implementing onModule by the type is not of Module type {}", plugin.name(), moduleClass); + continue; + } + method.setAccessible(true); + list.add(new OnModuleReference(moduleClass, method)); + } + if (!list.isEmpty()) { + onModuleReferences.put(plugin, list); + } + } + this.onModuleReferences = onModuleReferences.immutableMap(); } public ImmutableMap plugins() { @@ -80,6 +118,19 @@ public class PluginsService extends AbstractComponent { public void processModule(Module module) { for (Plugin plugin : plugins().values()) { plugin.processModule(module); + // see if there are onModule references + List references = onModuleReferences.get(plugin); + if (references != null) { + for (OnModuleReference reference : references) { + if (reference.moduleClass.isAssignableFrom(module.getClass())) { + try { + reference.onModuleMethod.invoke(plugin, module); + } catch (Exception e) { + logger.warn("plugin {}, failed to invoke custom onModule method", e, plugin.name()); + } + } + } + } } } From 9bdef666cb5fd0aeee5083d5f92ce127a51e1a15 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 15 Jan 2012 17:20:14 +0200 Subject: [PATCH 172/270] Plugin loading broken, closes #1610. --- .../org/elasticsearch/plugins/PluginsService.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/plugins/PluginsService.java b/src/main/java/org/elasticsearch/plugins/PluginsService.java index 1eae871796e..4f94f86983a 100644 --- a/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -245,18 +245,20 @@ public class PluginsService extends AbstractComponent { addURL.invoke(classLoader, pluginFile.toURI().toURL()); // gather files to add List libFiles = Lists.newArrayList(); - libFiles.addAll(Arrays.asList(pluginsFile.listFiles())); + if (pluginFile.listFiles() != null) { + libFiles.addAll(Arrays.asList(pluginFile.listFiles())); + } File libLocation = new File(pluginFile, "lib"); - if (libLocation.exists() && libLocation.isDirectory()) { + if (libLocation.exists() && libLocation.isDirectory() && libLocation.listFiles() != null) { libFiles.addAll(Arrays.asList(libLocation.listFiles())); } // if there are jars in it, add it as well - for (File jarToAdd : libFiles) { - if (!(jarToAdd.getName().endsWith(".jar") || jarToAdd.getName().endsWith(".zip"))) { + for (File libFile : libFiles) { + if (!(libFile.getName().endsWith(".jar") || libFile.getName().endsWith(".zip"))) { continue; } - addURL.invoke(classLoader, jarToAdd.toURI().toURL()); + addURL.invoke(classLoader, libFile.toURI().toURL()); } } catch (Exception e) { logger.warn("failed to add plugin [" + pluginFile + "]", e); From 21405f5aa49e77c345c187f3a4ec86e7f7dcd23f Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 15 Jan 2012 23:05:34 +0200 Subject: [PATCH 173/270] Highlighting: Add boundary_chars and boundary_max_size to control text boundaries with fast vector highlighter (term vector), closes #1614. --- .../SimpleBoundaryScanner2.java | 62 +++++++++++++++++++ .../search/highlight/HighlightPhase.java | 19 ++++-- .../highlight/HighlighterParseElement.java | 17 +++++ .../highlight/SearchContextHighlight.java | 19 ++++++ .../SourceScoreOrderFragmentsBuilder.java | 5 +- .../SourceSimpleFragmentsBuilder.java | 5 +- .../highlight/HighlighterSearchTests.java | 4 +- .../deps/lucene/VectorHighlighterTests.java | 2 +- 8 files changed, 120 insertions(+), 13 deletions(-) create mode 100644 src/main/java/org/elasticsearch/common/lucene/search/vectorhighlight/SimpleBoundaryScanner2.java diff --git a/src/main/java/org/elasticsearch/common/lucene/search/vectorhighlight/SimpleBoundaryScanner2.java b/src/main/java/org/elasticsearch/common/lucene/search/vectorhighlight/SimpleBoundaryScanner2.java new file mode 100644 index 00000000000..7a92ae7e327 --- /dev/null +++ b/src/main/java/org/elasticsearch/common/lucene/search/vectorhighlight/SimpleBoundaryScanner2.java @@ -0,0 +1,62 @@ +package org.elasticsearch.common.lucene.search.vectorhighlight; + +import gnu.trove.set.hash.TCharHashSet; +import org.apache.lucene.search.vectorhighlight.BoundaryScanner; + +/** + * A copy of Lucene {@link org.apache.lucene.search.vectorhighlight.SimpleBoundaryScanner}. + *

    + * Uses specialized char set to lookup boundary, and fixes a problem with start offset in the + * beginning of the text: https://issues.apache.org/jira/browse/LUCENE-3697 (which has a problem + * with multiple empty fields to highlight...). + */ +public class SimpleBoundaryScanner2 implements BoundaryScanner { + + public static final int DEFAULT_MAX_SCAN = 20; + public static final char[] DEFAULT_BOUNDARY_CHARS = {'.', ',', '!', '?', ' ', '\t', '\n'}; + + public static final SimpleBoundaryScanner2 DEFAULT = new SimpleBoundaryScanner2(); + + public int maxScan; + public TCharHashSet boundaryChars; + + public SimpleBoundaryScanner2() { + this(DEFAULT_MAX_SCAN, DEFAULT_BOUNDARY_CHARS); + } + + public SimpleBoundaryScanner2(int maxScan, char[] boundaryChars) { + this.maxScan = maxScan; + this.boundaryChars = new TCharHashSet(boundaryChars); + } + + public int findStartOffset(StringBuilder buffer, int start) { + // avoid illegal start offset + if (start > buffer.length() || start < 1) return start; + int offset, count = maxScan; + for (offset = start; offset > 0 && count > 0; count--) { + // found? + if (boundaryChars.contains(buffer.charAt(offset - 1))) return offset; + offset--; + } + // LUCENE-3697 + if (offset == 0) { + return 0; + } + // not found + return start; + } + + public int findEndOffset(StringBuilder buffer, int start) { + // avoid illegal start offset + if (start > buffer.length() || start < 0) return start; + int offset, count = maxScan; + //for( offset = start; offset <= buffer.length() && count > 0; count-- ){ + for (offset = start; offset < buffer.length() && count > 0; count--) { + // found? + if (boundaryChars.contains(buffer.charAt(offset))) return offset; + offset++; + } + // not found + return start; + } +} diff --git a/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java b/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java index a7006f79da4..6ddaa384067 100644 --- a/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java +++ b/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.io.FastStringReader; import org.elasticsearch.common.lucene.document.SingleFieldSelector; import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery; import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; +import org.elasticsearch.common.lucene.search.vectorhighlight.SimpleBoundaryScanner2; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; @@ -237,13 +238,19 @@ public class HighlightPhase implements FetchSubPhase { if (entry == null) { FragListBuilder fragListBuilder; FragmentsBuilder fragmentsBuilder; + + BoundaryScanner boundaryScanner = SimpleBoundaryScanner2.DEFAULT; + if (field.boundaryMaxScan() != SimpleBoundaryScanner2.DEFAULT_MAX_SCAN || field.boundaryChars() != SimpleBoundaryScanner2.DEFAULT_BOUNDARY_CHARS) { + boundaryScanner = new SimpleBoundaryScanner2(field.boundaryMaxScan(), field.boundaryChars()); + } + if (field.numberOfFragments() == 0) { fragListBuilder = new SingleFragListBuilder(); if (mapper.stored()) { - fragmentsBuilder = new SimpleFragmentsBuilder(field.preTags(), field.postTags()); + fragmentsBuilder = new SimpleFragmentsBuilder(field.preTags(), field.postTags(), boundaryScanner); } else { - fragmentsBuilder = new SourceSimpleFragmentsBuilder(mapper, context, field.preTags(), field.postTags()); + fragmentsBuilder = new SourceSimpleFragmentsBuilder(mapper, context, field.preTags(), field.postTags(), boundaryScanner); } } else { if (field.fragmentOffset() == -1) @@ -253,15 +260,15 @@ public class HighlightPhase implements FetchSubPhase { if (field.scoreOrdered()) { if (mapper.stored()) { - fragmentsBuilder = new ScoreOrderFragmentsBuilder(field.preTags(), field.postTags()); + fragmentsBuilder = new ScoreOrderFragmentsBuilder(field.preTags(), field.postTags(), boundaryScanner); } else { - fragmentsBuilder = new SourceScoreOrderFragmentsBuilder(mapper, context, field.preTags(), field.postTags()); + fragmentsBuilder = new SourceScoreOrderFragmentsBuilder(mapper, context, field.preTags(), field.postTags(), boundaryScanner); } } else { if (mapper.stored()) { - fragmentsBuilder = new SimpleFragmentsBuilder(field.preTags(), field.postTags()); + fragmentsBuilder = new SimpleFragmentsBuilder(field.preTags(), field.postTags(), boundaryScanner); } else { - fragmentsBuilder = new SourceSimpleFragmentsBuilder(mapper, context, field.preTags(), field.postTags()); + fragmentsBuilder = new SourceSimpleFragmentsBuilder(mapper, context, field.preTags(), field.postTags(), boundaryScanner); } } } diff --git a/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java b/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java index d204ec7b4a7..cbc104e8bd0 100644 --- a/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java +++ b/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.highlight; import com.google.common.collect.Lists; +import org.elasticsearch.common.lucene.search.vectorhighlight.SimpleBoundaryScanner2; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.SearchParseException; @@ -73,6 +74,8 @@ public class HighlighterParseElement implements SearchParseElement { int globalFragmentSize = 100; int globalNumOfFragments = 5; String globalEncoder = "default"; + int globalBoundaryMaxScan = SimpleBoundaryScanner2.DEFAULT_MAX_SCAN; + char[] globalBoundaryChars = SimpleBoundaryScanner2.DEFAULT_BOUNDARY_CHARS; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -110,6 +113,10 @@ public class HighlighterParseElement implements SearchParseElement { globalEncoder = parser.text(); } else if ("require_field_match".equals(topLevelFieldName) || "requireFieldMatch".equals(topLevelFieldName)) { globalRequireFieldMatch = parser.booleanValue(); + } else if ("boundary_max_scan".equals(topLevelFieldName) || "boundaryMaxScan".equals(topLevelFieldName)) { + globalBoundaryMaxScan = parser.intValue(); + } else if ("boundary_chars".equals(topLevelFieldName) || "boundaryChars".equals(topLevelFieldName)) { + globalBoundaryChars = parser.text().toCharArray(); } } else if (token == XContentParser.Token.START_OBJECT) { if ("fields".equals(topLevelFieldName)) { @@ -150,6 +157,10 @@ public class HighlighterParseElement implements SearchParseElement { field.scoreOrdered("score".equals(parser.text())); } else if ("require_field_match".equals(fieldName) || "requireFieldMatch".equals(fieldName)) { field.requireFieldMatch(parser.booleanValue()); + } else if ("boundary_max_scan".equals(topLevelFieldName) || "boundaryMaxScan".equals(topLevelFieldName)) { + field.boundaryMaxScan(parser.intValue()); + } else if ("boundary_chars".equals(topLevelFieldName) || "boundaryChars".equals(topLevelFieldName)) { + field.boundaryChars(parser.text().toCharArray()); } } } @@ -189,6 +200,12 @@ public class HighlighterParseElement implements SearchParseElement { if (field.requireFieldMatch() == null) { field.requireFieldMatch(globalRequireFieldMatch); } + if (field.boundaryMaxScan() == -1) { + field.boundaryMaxScan(globalBoundaryMaxScan); + } + if (field.boundaryChars() == null) { + field.boundaryChars(globalBoundaryChars); + } } context.highlight(new SearchContextHighlight(fields)); diff --git a/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java b/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java index 228781b4023..10d2878bf3f 100644 --- a/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java +++ b/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java @@ -58,6 +58,9 @@ public class SearchContextHighlight { private Boolean requireFieldMatch; + private int boundaryMaxScan = -1; + private char[] boundaryChars = null; + public Field(String field) { this.field = field; } @@ -137,5 +140,21 @@ public class SearchContextHighlight { public void requireFieldMatch(boolean requireFieldMatch) { this.requireFieldMatch = requireFieldMatch; } + + public int boundaryMaxScan() { + return boundaryMaxScan; + } + + public void boundaryMaxScan(int boundaryMaxScan) { + this.boundaryMaxScan = boundaryMaxScan; + } + + public char[] boundaryChars() { + return boundaryChars; + } + + public void boundaryChars(char[] boundaryChars) { + this.boundaryChars = boundaryChars; + } } } diff --git a/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java b/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java index a13e95704c5..0443616ca05 100644 --- a/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java +++ b/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.highlight.vectorhighlight; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.vectorhighlight.BoundaryScanner; import org.apache.lucene.search.vectorhighlight.ScoreOrderFragmentsBuilder; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.search.internal.SearchContext; @@ -39,8 +40,8 @@ public class SourceScoreOrderFragmentsBuilder extends ScoreOrderFragmentsBuilder private final SearchContext searchContext; public SourceScoreOrderFragmentsBuilder(FieldMapper mapper, SearchContext searchContext, - String[] preTags, String[] postTags) { - super(preTags, postTags); + String[] preTags, String[] postTags, BoundaryScanner boundaryScanner) { + super(preTags, postTags, boundaryScanner); this.mapper = mapper; this.searchContext = searchContext; } diff --git a/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java b/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java index 48bcc18f715..2cb35f01546 100644 --- a/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java +++ b/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.highlight.vectorhighlight; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.vectorhighlight.BoundaryScanner; import org.apache.lucene.search.vectorhighlight.SimpleFragmentsBuilder; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.search.internal.SearchContext; @@ -39,8 +40,8 @@ public class SourceSimpleFragmentsBuilder extends SimpleFragmentsBuilder { private final SearchContext searchContext; public SourceSimpleFragmentsBuilder(FieldMapper mapper, SearchContext searchContext, - String[] preTags, String[] postTags) { - super(preTags, postTags); + String[] preTags, String[] postTags, BoundaryScanner boundaryScanner) { + super(preTags, postTags, boundaryScanner); this.mapper = mapper; this.searchContext = searchContext; } diff --git a/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java index 6a3d5f9bab6..21811ecee0e 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java @@ -555,7 +555,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0)); SearchHit hit = search.hits().getAt(0); - assertThat(hit.highlightFields().get("title").fragments()[0], equalTo(" is a test ")); + assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("this is a test ")); // search on title.key and highlight on title search = client.prepareSearch() @@ -596,7 +596,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0)); SearchHit hit = search.hits().getAt(0); - assertThat(hit.highlightFields().get("title").fragments()[0], equalTo(" is a test ")); + assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("this is a test ")); // search on title.key and highlight on title.key search = client.prepareSearch() diff --git a/src/test/java/org/elasticsearch/test/unit/deps/lucene/VectorHighlighterTests.java b/src/test/java/org/elasticsearch/test/unit/deps/lucene/VectorHighlighterTests.java index 08039790693..fde13951bf8 100644 --- a/src/test/java/org/elasticsearch/test/unit/deps/lucene/VectorHighlighterTests.java +++ b/src/test/java/org/elasticsearch/test/unit/deps/lucene/VectorHighlighterTests.java @@ -60,7 +60,7 @@ public class VectorHighlighterTests { String fragment = highlighter.getBestFragment(highlighter.getFieldQuery(new TermQuery(new Term("content", "bad"))), reader, topDocs.scoreDocs[0].doc, "content", 30); assertThat(fragment, notNullValue()); - System.out.println(fragment); + assertThat(fragment, equalTo("e big bad dog ")); } @Test From 1f5a74a91af07bd37af6e424e8959967bc12e5ec Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 16 Jan 2012 13:40:49 +0200 Subject: [PATCH 174/270] lucene 3.5 verison --- src/main/java/org/elasticsearch/common/lucene/Lucene.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/src/main/java/org/elasticsearch/common/lucene/Lucene.java index e1620f5d1d2..48a8b6db8a7 100644 --- a/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -44,7 +44,7 @@ import java.util.Map; */ public class Lucene { - public static final Version VERSION = Version.LUCENE_34; + public static final Version VERSION = Version.LUCENE_35; public static final Version ANALYZER_VERSION = VERSION; public static final Version QUERYPARSER_VERSION = VERSION; @@ -59,6 +59,9 @@ public class Lucene { if (version == null) { return defaultVersion; } + if ("3.5".equals(version)) { + return Version.LUCENE_35; + } if ("3.4".equals(version)) { return Version.LUCENE_34; } From bddea0917030021cd3224e5cb9150a251cae1bd1 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 16 Jan 2012 13:48:31 +0200 Subject: [PATCH 175/270] /_status doc count of index wrong, closes #1615. --- .../admin/indices/status/DocsStatus.java | 18 +++++++++--------- .../admin/indices/status/ShardStatus.java | 14 ++++++-------- .../integration/nested/SimpleNestedTests.java | 14 +++++++------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/indices/status/DocsStatus.java b/src/main/java/org/elasticsearch/action/admin/indices/status/DocsStatus.java index aca640c2799..24d608cb83c 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/status/DocsStatus.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/status/DocsStatus.java @@ -24,49 +24,49 @@ package org.elasticsearch.action.admin.indices.status; */ public class DocsStatus { - int numDocs = 0; - int maxDoc = 0; - int deletedDocs = 0; + long numDocs = 0; + long maxDoc = 0; + long deletedDocs = 0; /** * The number of docs. */ - public int numDocs() { + public long numDocs() { return numDocs; } /** * The number of docs. */ - public int getNumDocs() { + public long getNumDocs() { return numDocs(); } /** * The max doc. */ - public int maxDoc() { + public long maxDoc() { return maxDoc; } /** * The max doc. */ - public int getMaxDoc() { + public long getMaxDoc() { return maxDoc(); } /** * The number of deleted docs in the index. */ - public int deletedDocs() { + public long deletedDocs() { return deletedDocs; } /** * The number of deleted docs in the index. */ - public int getDeletedDocs() { + public long getDeletedDocs() { return deletedDocs(); } } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/status/ShardStatus.java b/src/main/java/org/elasticsearch/action/admin/indices/status/ShardStatus.java index c905b777f7b..e47a9dd5981 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/status/ShardStatus.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/status/ShardStatus.java @@ -36,8 +36,6 @@ import static org.elasticsearch.common.unit.ByteSizeValue.readBytesSizeValue; /** * Shard instance (actual allocated shard) status. - * - * */ public class ShardStatus extends BroadcastShardOperationResponse { @@ -262,9 +260,9 @@ public class ShardStatus extends BroadcastShardOperationResponse { out.writeBoolean(false); } else { out.writeBoolean(true); - out.writeInt(docs.numDocs()); - out.writeInt(docs.maxDoc()); - out.writeInt(docs.deletedDocs()); + out.writeLong(docs.numDocs()); + out.writeLong(docs.maxDoc()); + out.writeLong(docs.deletedDocs()); } if (peerRecoveryStatus == null) { out.writeBoolean(false); @@ -335,9 +333,9 @@ public class ShardStatus extends BroadcastShardOperationResponse { translogOperations = in.readLong(); if (in.readBoolean()) { docs = new DocsStatus(); - docs.numDocs = in.readInt(); - docs.maxDoc = in.readInt(); - docs.deletedDocs = in.readInt(); + docs.numDocs = in.readLong(); + docs.maxDoc = in.readLong(); + docs.deletedDocs = in.readLong(); } if (in.readBoolean()) { peerRecoveryStatus = new PeerRecoveryStatus(PeerRecoveryStatus.Stage.fromValue(in.readByte()), diff --git a/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java b/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java index a474dcbd3e2..5e09557f6ef 100644 --- a/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java +++ b/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java @@ -104,7 +104,7 @@ public class SimpleNestedTests extends AbstractNodesTests { // check the numDocs IndicesStatusResponse statusResponse = client.admin().indices().prepareStatus().execute().actionGet(); - assertThat(statusResponse.index("test").docs().numDocs(), equalTo(3)); + assertThat(statusResponse.index("test").docs().numDocs(), equalTo(3l)); // check that _all is working on nested docs searchResponse = client.prepareSearch("test").setQuery(termQuery("_all", "n_value1_1")).execute().actionGet(); @@ -147,7 +147,7 @@ public class SimpleNestedTests extends AbstractNodesTests { client.admin().indices().prepareFlush().setRefresh(true).execute().actionGet(); statusResponse = client.admin().indices().prepareStatus().execute().actionGet(); - assertThat(statusResponse.index("test").docs().numDocs(), equalTo(6)); + assertThat(statusResponse.index("test").docs().numDocs(), equalTo(6l)); searchResponse = client.prepareSearch("test").setQuery(nestedQuery("nested1", boolQuery().must(termQuery("nested1.n_field1", "n_value1_1")).must(termQuery("nested1.n_field2", "n_value2_1")))).execute().actionGet(); @@ -174,7 +174,7 @@ public class SimpleNestedTests extends AbstractNodesTests { client.admin().indices().prepareFlush().setRefresh(true).execute().actionGet(); statusResponse = client.admin().indices().prepareStatus().execute().actionGet(); - assertThat(statusResponse.index("test").docs().numDocs(), equalTo(3)); + assertThat(statusResponse.index("test").docs().numDocs(), equalTo(3l)); searchResponse = client.prepareSearch("test").setQuery(nestedQuery("nested1", termQuery("nested1.n_field1", "n_value1_1"))).execute().actionGet(); assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0)); @@ -230,12 +230,12 @@ public class SimpleNestedTests extends AbstractNodesTests { client.admin().indices().prepareFlush().setRefresh(true).execute().actionGet(); IndicesStatusResponse statusResponse = client.admin().indices().prepareStatus().execute().actionGet(); - assertThat(statusResponse.index("test").docs().numDocs(), equalTo(total * 3)); + assertThat(statusResponse.index("test").docs().numDocs(), equalTo(total * 3l)); client.prepareDeleteByQuery("test").setQuery(QueryBuilders.idsQuery("type1").ids(Integer.toString(docToDelete))).execute().actionGet(); client.admin().indices().prepareFlush().setRefresh(true).execute().actionGet(); statusResponse = client.admin().indices().prepareStatus().execute().actionGet(); - assertThat(statusResponse.index("test").docs().numDocs(), equalTo((total * 3) - 3)); + assertThat(statusResponse.index("test").docs().numDocs(), equalTo((total * 3l) - 3)); for (int i = 0; i < total; i++) { assertThat(client.prepareGet("test", "type1", Integer.toString(i)).execute().actionGet().exists(), equalTo(i != docToDelete)); @@ -257,7 +257,7 @@ public class SimpleNestedTests extends AbstractNodesTests { noChildrenNestedDeleteByQuery(3, 2); } - private void noChildrenNestedDeleteByQuery(int total, int docToDelete) throws Exception { + private void noChildrenNestedDeleteByQuery(long total, int docToDelete) throws Exception { client.admin().indices().prepareDelete().execute().actionGet(); client.admin().indices().prepareCreate("test") @@ -324,7 +324,7 @@ public class SimpleNestedTests extends AbstractNodesTests { // check the numDocs IndicesStatusResponse statusResponse = client.admin().indices().prepareStatus().execute().actionGet(); - assertThat(statusResponse.index("test").docs().numDocs(), equalTo(7)); + assertThat(statusResponse.index("test").docs().numDocs(), equalTo(7l)); // do some multi nested queries SearchResponse searchResponse = client.prepareSearch("test").setQuery(nestedQuery("nested1", From 56a9c66df183ea4e9e2cfb8861877283d4048304 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 16 Jan 2012 21:09:07 +0200 Subject: [PATCH 176/270] improve a bit the speed of adding new fields as part of mapping change / parse --- .../index/mapper/FieldMappers.java | 74 +++++++------------ .../index/mapper/MapperService.java | 43 +++++------ 2 files changed, 47 insertions(+), 70 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/mapper/FieldMappers.java b/src/main/java/org/elasticsearch/index/mapper/FieldMappers.java index c8f21b56519..9ba7308313f 100644 --- a/src/main/java/org/elasticsearch/index/mapper/FieldMappers.java +++ b/src/main/java/org/elasticsearch/index/mapper/FieldMappers.java @@ -19,10 +19,9 @@ package org.elasticsearch.index.mapper; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterators; -import com.google.common.collect.UnmodifiableIterator; - +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; import java.util.List; /** @@ -30,84 +29,61 @@ import java.util.List; */ public class FieldMappers implements Iterable { - private final ImmutableList fieldMappers; + private final FieldMapper[] fieldMappers; + private final List fieldMappersAsList; public FieldMappers() { - this.fieldMappers = ImmutableList.of(); + this.fieldMappers = new FieldMapper[0]; + this.fieldMappersAsList = Arrays.asList(fieldMappers); } public FieldMappers(FieldMapper fieldMapper) { - this(new FieldMapper[]{fieldMapper}); + this.fieldMappers = new FieldMapper[]{fieldMapper}; + this.fieldMappersAsList = Arrays.asList(this.fieldMappers); } - public FieldMappers(FieldMapper[] fieldMappers) { - if (fieldMappers == null) { - fieldMappers = new FieldMapper[0]; - } - this.fieldMappers = ImmutableList.copyOf(Iterators.forArray(fieldMappers)); - } - - public FieldMappers(ImmutableList fieldMappers) { + private FieldMappers(FieldMapper[] fieldMappers) { this.fieldMappers = fieldMappers; + this.fieldMappersAsList = Arrays.asList(this.fieldMappers); } public FieldMapper mapper() { - if (fieldMappers.isEmpty()) { + if (fieldMappers.length == 0) { return null; } - return fieldMappers.get(0); + return fieldMappers[0]; } public boolean isEmpty() { - return fieldMappers.isEmpty(); + return fieldMappers.length == 0; } - public ImmutableList mappers() { - return this.fieldMappers; + public List mappers() { + return this.fieldMappersAsList; } @Override - public UnmodifiableIterator iterator() { - return fieldMappers.iterator(); + public Iterator iterator() { + return fieldMappersAsList.iterator(); } /** * Concats and returns a new {@link FieldMappers}. */ public FieldMappers concat(FieldMapper mapper) { - return new FieldMappers(new ImmutableList.Builder().addAll(fieldMappers).add(mapper).build()); - } - - /** - * Concats and returns a new {@link FieldMappers}. - */ - public FieldMappers concat(FieldMappers mappers) { - return new FieldMappers(new ImmutableList.Builder().addAll(fieldMappers).addAll(mappers).build()); - } - - public FieldMappers remove(List mappers) { - ImmutableList.Builder builder = new ImmutableList.Builder(); - for (FieldMapper fieldMapper : fieldMappers) { - boolean found = false; - for (FieldMapper mapper : mappers) { - if (fieldMapper.equals(mapper)) { // identify equality - found = true; - } - } - if (!found) { - builder.add(fieldMapper); - } - } - return new FieldMappers(builder.build()); + FieldMapper[] newMappers = new FieldMapper[fieldMappers.length + 1]; + System.arraycopy(fieldMappers, 0, newMappers, 0, fieldMappers.length); + newMappers[fieldMappers.length] = mapper; + return new FieldMappers(newMappers); } public FieldMappers remove(FieldMapper mapper) { - ImmutableList.Builder builder = new ImmutableList.Builder(); + ArrayList list = new ArrayList(fieldMappers.length); for (FieldMapper fieldMapper : fieldMappers) { if (!fieldMapper.equals(mapper)) { // identify equality - builder.add(fieldMapper); + list.add(fieldMapper); } } - return new FieldMappers(builder.build()); + return new FieldMappers(list.toArray(new FieldMapper[list.size()])); } } diff --git a/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 5ace3cb7a40..c65fe4ae801 100644 --- a/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterators; import com.google.common.collect.Sets; import com.google.common.collect.UnmodifiableIterator; import org.apache.lucene.analysis.Analyzer; @@ -78,14 +79,14 @@ public class MapperService extends AbstractIndexComponent implements Iterable mappers = ImmutableMap.of(); + private volatile Map mappers = ImmutableMap.of(); private final Object mutex = new Object(); - private volatile ImmutableMap nameFieldMappers = ImmutableMap.of(); - private volatile ImmutableMap indexNameFieldMappers = ImmutableMap.of(); - private volatile ImmutableMap fullNameFieldMappers = ImmutableMap.of(); - private volatile ImmutableMap objectMappers = ImmutableMap.of(); + private volatile Map nameFieldMappers = ImmutableMap.of(); + private volatile Map indexNameFieldMappers = ImmutableMap.of(); + private volatile Map fullNameFieldMappers = ImmutableMap.of(); + private volatile Map objectMappers = ImmutableMap.of(); private boolean hasNested = false; // updated dynamically to true when a nested object is added private final DocumentMapperParser documentParser; @@ -146,7 +147,7 @@ public class MapperService extends AbstractIndexComponent implements Iterable iterator() { - return mappers.values().iterator(); + return Iterators.unmodifiableIterator(mappers.values().iterator()); } public AnalysisService analysisService() { @@ -164,7 +165,7 @@ public class MapperService extends AbstractIndexComponent implements Iterable Date: Tue, 17 Jan 2012 10:35:16 +0100 Subject: [PATCH 177/270] fix bug in TTL handling where default TTL value was not set properly --- .../index/mapper/internal/TTLFieldMapper.java | 1 + .../test/integration/ttl/SimpleTTLTests.java | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java index aa179f1da2f..d9f672e4c72 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java @@ -182,6 +182,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R long ttl = context.sourceToParse().ttl(); if (ttl <= 0 && defaultTTL > 0) { // no ttl provided so we use the default value ttl = defaultTTL; + context.sourceToParse().ttl(ttl); } if (ttl > 0) { // a ttl has been provided either externally or in the _source long timestamp = context.sourceToParse().timestamp(); diff --git a/src/test/java/org/elasticsearch/test/integration/ttl/SimpleTTLTests.java b/src/test/java/org/elasticsearch/test/integration/ttl/SimpleTTLTests.java index 3b8f00c041b..ed8736660ed 100644 --- a/src/test/java/org/elasticsearch/test/integration/ttl/SimpleTTLTests.java +++ b/src/test/java/org/elasticsearch/test/integration/ttl/SimpleTTLTests.java @@ -72,13 +72,22 @@ public class SimpleTTLTests extends AbstractNodesTests { .startObject("_ttl").field("enabled", true).field("store", "yes").endObject() .endObject() .endObject()) + .addMapping("type2", XContentFactory.jsonBuilder() + .startObject() + .startObject("type2") + .startObject("_timestamp").field("enabled", true).field("store", "yes").endObject() + .startObject("_ttl").field("enabled", true).field("store", "yes").field("default", "1d").endObject() + .endObject() + .endObject()) .execute().actionGet(); client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); long providedTTLValue = 3000; logger.info("--> checking ttl"); - // Index one doc without routing and one doc with routing + // Index one doc without routing, one doc with routing, one doc with not TTL and no default and one doc with default TTL client.prepareIndex("test", "type1", "1").setSource("field1", "value1").setTTL(providedTTLValue).setRefresh(true).execute().actionGet(); client.prepareIndex("test", "type1", "with_routing").setSource("field1", "value1").setTTL(providedTTLValue).setRouting("routing").setRefresh(true).execute().actionGet(); + client.prepareIndex("test", "type1", "no_ttl").setSource("field1", "value1").execute().actionGet(); + client.prepareIndex("test", "type2", "default_ttl").setSource("field1", "value1").execute().actionGet(); long now = System.currentTimeMillis(); // realtime get check @@ -106,6 +115,14 @@ public class SimpleTTLTests extends AbstractNodesTests { assertThat(ttl0, greaterThan(0L)); assertThat(ttl0, lessThan(providedTTLValue - (now1 - now))); + // no TTL provided so no TTL fetched + getResponse = client.prepareGet("test", "type1", "no_ttl").setFields("_ttl").setRealtime(true).execute().actionGet(); + assertThat(getResponse.field("_ttl"), nullValue()); + // no TTL provided make sure it has default TTL + getResponse = client.prepareGet("test", "type2", "default_ttl").setFields("_ttl").setRealtime(true).execute().actionGet(); + ttl0 = ((Number) getResponse.field("_ttl").value()).longValue(); + assertThat(ttl0, greaterThan(0L)); + logger.info("--> checking purger"); // make sure the purger has done its job for all indexed docs that are expired long shouldBeExpiredDate = now + providedTTLValue + purgeInterval + 2000; From 0d2d0936a7d3d7ec0485c5c2370464c5df49ec14 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 17 Jan 2012 11:47:21 +0200 Subject: [PATCH 178/270] support one, and all, all-1 for index.recovery.initial_shards --- .../elasticsearch/gateway/local/LocalGatewayAllocator.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java b/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java index ba17f3f96e4..ddb2fca430a 100644 --- a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java +++ b/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java @@ -164,9 +164,11 @@ public class LocalGatewayAllocator extends AbstractComponent implements GatewayA if (indexMetaData.numberOfReplicas() > 2) { requiredAllocation = ((1 + indexMetaData.numberOfReplicas()) / 2); } - } else if ("full".equals(initialShards)) { + } else if ("one".equals(initialShards)) { + requiredAllocation = 1; + } else if ("full".equals(initialShards) || "all".equals(initialShards)) { requiredAllocation = indexMetaData.numberOfReplicas() + 1; - } else if ("full-1".equals(initialShards)) { + } else if ("full-1".equals(initialShards) || "all-1".equals(initialShards)) { if (indexMetaData.numberOfReplicas() > 1) { requiredAllocation = indexMetaData.numberOfReplicas(); } From c2ef91afe6e7ea03ab112cb94415ac548f08ee1a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 17 Jan 2012 12:58:30 +0200 Subject: [PATCH 179/270] better logging when failing to load field data to memory due to OOM --- .../support/AbstractConcurrentMapFieldDataCache.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java b/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java index dc81261c5cf..1bf1850f291 100644 --- a/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java +++ b/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java @@ -118,8 +118,15 @@ public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexC synchronized (fieldDataCache) { fieldData = fieldDataCache.getIfPresent(fieldName); if (fieldData == null) { - fieldData = FieldData.load(type, reader, fieldName); - fieldDataCache.put(fieldName, fieldData); + try { + fieldData = FieldData.load(type, reader, fieldName); + fieldDataCache.put(fieldName, fieldData); + } catch (OutOfMemoryError e) { + logger.warn("loading field [" + fieldName + "] caused out of memory failure", e); + final OutOfMemoryError outOfMemoryError = new OutOfMemoryError("loading field [" + fieldName + "] caused out of memory failure"); + outOfMemoryError.initCause(e); + throw outOfMemoryError; + } } } } From 801c709b4243ee7089c093daf47ca9b2a0bd1878 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 18 Jan 2012 01:02:55 +0200 Subject: [PATCH 180/270] test with local gateway --- .../test/stress/manyindices/ManyIndicesStressTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/elasticsearch/test/stress/manyindices/ManyIndicesStressTest.java b/src/test/java/org/elasticsearch/test/stress/manyindices/ManyIndicesStressTest.java index 5916647e10d..d78fd04866a 100644 --- a/src/test/java/org/elasticsearch/test/stress/manyindices/ManyIndicesStressTest.java +++ b/src/test/java/org/elasticsearch/test/stress/manyindices/ManyIndicesStressTest.java @@ -47,7 +47,7 @@ public class ManyIndicesStressTest { Settings settings = ImmutableSettings.settingsBuilder() .put("index.shard.check_on_startup", false) - .put("gateway.type", "fs") + .put("gateway.type", "local") .put("index.number_of_shards", 1) .build(); Node node = NodeBuilder.nodeBuilder().settings(settings).node(); From 534f487de3edd7b6a4ca6d6aed7bdbc5b0ae2b7a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 18 Jan 2012 01:08:35 +0200 Subject: [PATCH 181/270] Local Gateway: Move shard state to be stored under each shard, and not globally under _state, closes #1618. --- .../elasticsearch/env/NodeEnvironment.java | 32 ++ .../gateway/local/LocalGateway.java | 258 +--------- .../gateway/local/LocalGatewayAllocator.java | 1 + .../gateway/local/LocalGatewayModule.java | 3 + .../local/LocalGatewayStartedShards.java | 185 ------- .../state/shards/LocalGatewayShardsState.java | 472 ++++++++++++++++++ .../local/state/shards/ShardStateInfo.java | 31 ++ ...ransportNodesListGatewayStartedShards.java | 17 +- .../SimpleRecoveryLocalGatewayTests.java | 4 +- 9 files changed, 568 insertions(+), 435 deletions(-) delete mode 100644 src/main/java/org/elasticsearch/gateway/local/LocalGatewayStartedShards.java create mode 100644 src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java create mode 100644 src/main/java/org/elasticsearch/gateway/local/state/shards/ShardStateInfo.java rename src/main/java/org/elasticsearch/gateway/local/{ => state/shards}/TransportNodesListGatewayStartedShards.java (94%) diff --git a/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/src/main/java/org/elasticsearch/env/NodeEnvironment.java index c28043ed2df..fa71a28a665 100644 --- a/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -19,6 +19,7 @@ package org.elasticsearch.env; +import com.google.common.collect.Sets; import org.apache.lucene.store.Lock; import org.apache.lucene.store.NativeFSLockFactory; import org.elasticsearch.ElasticSearchIllegalStateException; @@ -34,6 +35,7 @@ import org.elasticsearch.index.shard.ShardId; import java.io.File; import java.io.IOException; import java.util.Arrays; +import java.util.Set; /** * @@ -175,6 +177,36 @@ public class NodeEnvironment extends AbstractComponent { return shardLocations; } + public Set findAllShardIds() throws Exception { + if (nodeFiles == null || locks == null) { + throw new ElasticSearchIllegalStateException("node is not configured to store local location"); + } + Set shardIds = Sets.newHashSet(); + for (File indicesLocation : nodeIndicesLocations) { + File[] indicesList = indicesLocation.listFiles(); + if (indicesList == null) { + continue; + } + for (File indexLocation : indicesList) { + if (!indexLocation.isDirectory()) { + continue; + } + String indexName = indexLocation.getName(); + File[] shardsList = indexLocation.listFiles(); + if (shardsList == null) { + continue; + } + for (File shardLocation : shardsList) { + if (!shardLocation.isDirectory()) { + continue; + } + shardIds.add(new ShardId(indexName, Integer.parseInt(shardLocation.getName()))); + } + } + } + return shardIds; + } + public void close() { if (locks != null) { for (Lock lock : locks) { diff --git a/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java b/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java index 1fae23024c0..79d8eb38034 100644 --- a/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java +++ b/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java @@ -28,7 +28,6 @@ import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.routing.*; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.compress.lzf.LZF; import org.elasticsearch.common.inject.Inject; @@ -42,8 +41,8 @@ import org.elasticsearch.common.xcontent.*; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.gateway.Gateway; import org.elasticsearch.gateway.GatewayException; +import org.elasticsearch.gateway.local.state.shards.LocalGatewayShardsState; import org.elasticsearch.index.gateway.local.LocalIndexGatewayModule; -import org.elasticsearch.index.shard.ShardId; import java.io.*; import java.util.Set; @@ -58,24 +57,19 @@ import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadF */ public class LocalGateway extends AbstractLifecycleComponent implements Gateway, ClusterStateListener { - private boolean requiresStatePersistence; - private final ClusterService clusterService; private final NodeEnvironment nodeEnv; + private final LocalGatewayShardsState shardsState; + private final TransportNodesListGatewayMetaState listGatewayMetaState; - private final TransportNodesListGatewayStartedShards listGatewayStartedShards; - - private final boolean compress; private final boolean prettyPrint; private volatile LocalGatewayMetaState currentMetaState; - private volatile LocalGatewayStartedShards currentStartedShards; - private volatile ExecutorService executor; private volatile boolean initialized = false; @@ -83,13 +77,14 @@ public class LocalGateway extends AbstractLifecycleComponent implements private volatile boolean metaDataPersistedAtLeastOnce = false; @Inject - public LocalGateway(Settings settings, ClusterService clusterService, NodeEnvironment nodeEnv, - TransportNodesListGatewayMetaState listGatewayMetaState, TransportNodesListGatewayStartedShards listGatewayStartedShards) { + public LocalGateway(Settings settings, ClusterService clusterService, NodeEnvironment nodeEnv, LocalGatewayShardsState shardsState, + TransportNodesListGatewayMetaState listGatewayMetaState) { super(settings); this.clusterService = clusterService; this.nodeEnv = nodeEnv; this.listGatewayMetaState = listGatewayMetaState.initGateway(this); - this.listGatewayStartedShards = listGatewayStartedShards.initGateway(this); + + this.shardsState = shardsState; this.compress = componentSettings.getAsBoolean("compress", true); this.prettyPrint = componentSettings.getAsBoolean("pretty", false); @@ -105,16 +100,11 @@ public class LocalGateway extends AbstractLifecycleComponent implements return this.currentMetaState; } - public LocalGatewayStartedShards currentStartedShards() { - lazyInitialize(); - return this.currentStartedShards; - } - @Override protected void doStart() throws ElasticSearchException { this.executor = newSingleThreadExecutor(daemonThreadFactory(settings, "gateway")); lazyInitialize(); - clusterService.add(this); + clusterService.addLast(this); } @Override @@ -178,10 +168,6 @@ public class LocalGateway extends AbstractLifecycleComponent implements @Override public void clusterChanged(final ClusterChangedEvent event) { - if (!requiresStatePersistence) { - return; - } - // nothing to do until we actually recover from the gateway or any other block indicates we need to disable persistency if (event.state().blocks().disableStatePersistence()) { return; @@ -192,50 +178,7 @@ public class LocalGateway extends AbstractLifecycleComponent implements executor.execute(new LoggingRunnable(logger, new PersistMetaData(event))); } - if (event.state().nodes().localNode().dataNode() && event.routingTableChanged()) { - LocalGatewayStartedShards.Builder builder = LocalGatewayStartedShards.builder(); - if (currentStartedShards != null) { - builder.state(currentStartedShards); - } - builder.version(event.state().version()); - - boolean changed = false; - - // remove from the current state all the shards that are primary and started somewhere, we won't need them anymore - // and if they are still here, we will add them in the next phase - - // Also note, this works well when closing an index, since a closed index will have no routing shards entries - // so they won't get removed (we want to keep the fact that those shards are allocated on this node if needed) - for (IndexRoutingTable indexRoutingTable : event.state().routingTable()) { - for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { - if (indexShardRoutingTable.countWithState(ShardRoutingState.STARTED) == indexShardRoutingTable.size()) { - changed |= builder.remove(indexShardRoutingTable.shardId()); - } - } - } - // remove deleted indices from the started shards - for (ShardId shardId : builder.build().shards().keySet()) { - if (!event.state().metaData().hasIndex(shardId.index().name())) { - changed |= builder.remove(shardId); - } - } - // now, add all the ones that are active and on this node - RoutingNode routingNode = event.state().readOnlyRoutingNodes().node(event.state().nodes().localNodeId()); - if (routingNode != null) { - // out node is not in play yet... - for (MutableShardRouting shardRouting : routingNode) { - if (shardRouting.active()) { - changed |= builder.put(shardRouting.shardId(), shardRouting.version()); - } - } - } - - // only write if something changed... - if (changed) { - final LocalGatewayStartedShards stateToWrite = builder.build(); - executor.execute(new LoggingRunnable(logger, new PersistShards(event, stateToWrite))); - } - } + shardsState.clusterChanged(event); } /** @@ -251,84 +194,21 @@ public class LocalGateway extends AbstractLifecycleComponent implements } initialized = true; - // if this is not a possible master node or data node, bail, we won't save anything here... - if (!clusterService.localNode().masterNode() && !clusterService.localNode().dataNode()) { - requiresStatePersistence = false; - } else { - // create the location where the state will be stored - // TODO: we might want to persist states on all data locations - requiresStatePersistence = true; - - if (clusterService.localNode().masterNode()) { - try { - File latest = findLatestMetaStateVersion(); - if (latest != null) { - logger.debug("[find_latest_state]: loading metadata from [{}]", latest.getAbsolutePath()); - this.currentMetaState = readMetaState(Streams.copyToByteArray(new FileInputStream(latest))); - } else { - logger.debug("[find_latest_state]: no metadata state loaded"); - } - } catch (Exception e) { - logger.warn("failed to read local state (metadata)", e); - } - } - - if (clusterService.localNode().dataNode()) { - try { - File latest = findLatestStartedShardsVersion(); - if (latest != null) { - logger.debug("[find_latest_state]: loading started shards from [{}]", latest.getAbsolutePath()); - this.currentStartedShards = readStartedShards(Streams.copyToByteArray(new FileInputStream(latest))); - } else { - logger.debug("[find_latest_state]: no started shards loaded"); - } - } catch (Exception e) { - logger.warn("failed to read local state (started shards)", e); + if (clusterService.localNode().masterNode()) { + try { + File latest = findLatestMetaStateVersion(); + if (latest != null) { + logger.debug("[find_latest_state]: loading metadata from [{}]", latest.getAbsolutePath()); + this.currentMetaState = readMetaState(Streams.copyToByteArray(new FileInputStream(latest))); + } else { + logger.debug("[find_latest_state]: no metadata state loaded"); } + } catch (Exception e) { + logger.warn("failed to read local state (metadata)", e); } } } - private File findLatestStartedShardsVersion() throws IOException { - long index = -1; - File latest = null; - for (File dataLocation : nodeEnv.nodeDataLocations()) { - File stateLocation = new File(dataLocation, "_state"); - if (!stateLocation.exists()) { - continue; - } - File[] stateFiles = stateLocation.listFiles(); - if (stateFiles == null) { - continue; - } - for (File stateFile : stateFiles) { - if (logger.isTraceEnabled()) { - logger.trace("[find_latest_state]: processing [" + stateFile.getName() + "]"); - } - String name = stateFile.getName(); - if (!name.startsWith("shards-")) { - continue; - } - long fileIndex = Long.parseLong(name.substring(name.indexOf('-') + 1)); - if (fileIndex >= index) { - // try and read the meta data - try { - byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile)); - if (data.length == 0) { - logger.debug("[find_latest_state]: not data for [" + name + "], ignoring..."); - } - readStartedShards(data); - index = fileIndex; - latest = stateFile; - } catch (IOException e) { - logger.warn("[find_latest_state]: failed to read state from [" + name + "], ignoring...", e); - } - } - } - } - return latest; - } - private File findLatestMetaStateVersion() throws IOException { long index = -1; File latest = null; @@ -388,24 +268,6 @@ public class LocalGateway extends AbstractLifecycleComponent implements } } - private LocalGatewayStartedShards readStartedShards(byte[] data) throws IOException { - XContentParser parser = null; - try { - if (LZF.isCompressed(data)) { - BytesStreamInput siBytes = new BytesStreamInput(data, false); - LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes); - parser = XContentFactory.xContent(XContentType.JSON).createParser(siLzf); - } else { - parser = XContentFactory.xContent(XContentType.JSON).createParser(data); - } - return LocalGatewayStartedShards.Builder.fromXContent(parser); - } finally { - if (parser != null) { - parser.close(); - } - } - } - class PersistMetaData implements Runnable { private final ClusterChangedEvent event; @@ -493,86 +355,4 @@ public class LocalGateway extends AbstractLifecycleComponent implements } } } - - class PersistShards implements Runnable { - private final ClusterChangedEvent event; - private final LocalGatewayStartedShards stateToWrite; - - public PersistShards(ClusterChangedEvent event, LocalGatewayStartedShards stateToWrite) { - this.event = event; - this.stateToWrite = stateToWrite; - } - - @Override - public void run() { - - CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry(); - try { - StreamOutput streamOutput; - try { - if (compress) { - streamOutput = cachedEntry.cachedLZFBytes(); - } else { - streamOutput = cachedEntry.cachedBytes(); - } - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON, streamOutput); - if (prettyPrint) { - xContentBuilder.prettyPrint(); - } - xContentBuilder.startObject(); - LocalGatewayStartedShards.Builder.toXContent(stateToWrite, xContentBuilder, ToXContent.EMPTY_PARAMS); - xContentBuilder.endObject(); - xContentBuilder.close(); - } catch (Exception e) { - logger.warn("failed to serialize local gateway shard states", e); - return; - } - - boolean serializedAtLeastOnce = false; - for (File dataLocation : nodeEnv.nodeDataLocations()) { - File stateLocation = new File(dataLocation, "_state"); - if (!stateLocation.exists()) { - FileSystemUtils.mkdirs(stateLocation); - } - File stateFile = new File(stateLocation, "shards-" + event.state().version()); - FileOutputStream fos = null; - try { - fos = new FileOutputStream(stateFile); - fos.write(cachedEntry.bytes().underlyingBytes(), 0, cachedEntry.bytes().size()); - fos.getChannel().force(true); - serializedAtLeastOnce = true; - } catch (Exception e) { - logger.warn("failed to write local gateway shards state to {}", e, stateFile); - } finally { - Closeables.closeQuietly(fos); - } - } - - if (serializedAtLeastOnce) { - currentStartedShards = stateToWrite; - - // delete all the other files - for (File dataLocation : nodeEnv.nodeDataLocations()) { - File stateLocation = new File(dataLocation, "_state"); - if (!stateLocation.exists()) { - continue; - } - File[] files = stateLocation.listFiles(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.startsWith("shards-") && !name.equals("shards-" + event.state().version()); - } - }); - if (files != null) { - for (File file : files) { - file.delete(); - } - } - } - } - } finally { - CachedStreamOutput.pushEntry(cachedEntry); - } - } - } } diff --git a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java b/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java index ddb2fca430a..ac9b4171e09 100644 --- a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java +++ b/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java @@ -42,6 +42,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.gateway.local.state.shards.TransportNodesListGatewayStartedShards; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData; diff --git a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayModule.java b/src/main/java/org/elasticsearch/gateway/local/LocalGatewayModule.java index f3ae64bc2f9..c2f2b727950 100644 --- a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayModule.java +++ b/src/main/java/org/elasticsearch/gateway/local/LocalGatewayModule.java @@ -24,6 +24,8 @@ import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.inject.PreProcessModule; import org.elasticsearch.gateway.Gateway; +import org.elasticsearch.gateway.local.state.shards.LocalGatewayShardsState; +import org.elasticsearch.gateway.local.state.shards.TransportNodesListGatewayStartedShards; /** * @@ -33,6 +35,7 @@ public class LocalGatewayModule extends AbstractModule implements PreProcessModu @Override protected void configure() { bind(Gateway.class).to(LocalGateway.class).asEagerSingleton(); + bind(LocalGatewayShardsState.class).asEagerSingleton(); bind(TransportNodesListGatewayMetaState.class).asEagerSingleton(); bind(TransportNodesListGatewayStartedShards.class).asEagerSingleton(); } diff --git a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayStartedShards.java b/src/main/java/org/elasticsearch/gateway/local/LocalGatewayStartedShards.java deleted file mode 100644 index 33afc114ad3..00000000000 --- a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayStartedShards.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.gateway.local; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.shard.ShardId; - -import java.io.IOException; -import java.util.Map; - -/** - * - */ -public class LocalGatewayStartedShards { - - private final long version; - - private final ImmutableMap shards; - - public LocalGatewayStartedShards(long version, Map shards) { - this.version = version; - this.shards = ImmutableMap.copyOf(shards); - } - - public long version() { - return version; - } - - public ImmutableMap shards() { - return shards; - } - - public static Builder builder() { - return new Builder(); - } - - public static class Builder { - - private long version; - - private Map shards = Maps.newHashMap(); - - public Builder state(LocalGatewayStartedShards state) { - this.version = state.version(); - this.shards.putAll(state.shards); - return this; - } - - public Builder version(long version) { - this.version = version; - return this; - } - - /** - * Returns true if something really changed. - */ - public boolean remove(ShardId shardId) { - return shards.remove(shardId) != null; - } - - /** - * Returns true if something really changed. - */ - public boolean put(ShardId shardId, long version) { - Long lVersion = shards.get(shardId); - if (lVersion != null && lVersion == version) { - return false; - } - this.shards.put(shardId, version); - return true; - } - - public LocalGatewayStartedShards build() { - return new LocalGatewayStartedShards(version, shards); - } - - public static void toXContent(LocalGatewayStartedShards state, XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startObject("state"); - - builder.field("version", state.version()); - - builder.startArray("shards"); - for (Map.Entry entry : state.shards.entrySet()) { - builder.startObject(); - builder.field("index", entry.getKey().index().name()); - builder.field("id", entry.getKey().id()); - builder.field("version", entry.getValue()); - builder.endObject(); - } - builder.endArray(); - - builder.endObject(); - } - - public static LocalGatewayStartedShards fromXContent(XContentParser parser) throws IOException { - Builder builder = new Builder(); - - String currentFieldName = null; - XContentParser.Token token = parser.nextToken(); - if (token == null) { - // no data... - return builder.build(); - } - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token == XContentParser.Token.START_ARRAY) { - if ("shards".equals(currentFieldName)) { - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - if (token == XContentParser.Token.START_OBJECT) { - String shardIndex = null; - int shardId = -1; - long version = -1; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token.isValue()) { - if ("index".equals(currentFieldName)) { - shardIndex = parser.text(); - } else if ("id".equals(currentFieldName)) { - shardId = parser.intValue(); - } else if ("version".equals(currentFieldName)) { - version = parser.longValue(); - } - } - } - builder.shards.put(new ShardId(shardIndex, shardId), version); - } - } - } - } else if (token.isValue()) { - if ("version".equals(currentFieldName)) { - builder.version = parser.longValue(); - } - } - } - - return builder.build(); - } - - public static LocalGatewayStartedShards readFrom(StreamInput in) throws IOException { - LocalGatewayStartedShards.Builder builder = new Builder(); - builder.version = in.readLong(); - int size = in.readVInt(); - for (int i = 0; i < size; i++) { - builder.shards.put(ShardId.readShardId(in), in.readLong()); - } - return builder.build(); - } - - public static void writeTo(LocalGatewayStartedShards state, StreamOutput out) throws IOException { - out.writeLong(state.version()); - - out.writeVInt(state.shards.size()); - for (Map.Entry entry : state.shards.entrySet()) { - entry.getKey().writeTo(out); - out.writeLong(entry.getValue()); - } - } - } - -} diff --git a/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java b/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java new file mode 100644 index 00000000000..8b7cdee383b --- /dev/null +++ b/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java @@ -0,0 +1,472 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gateway.local.state.shards; + +import com.google.common.collect.Maps; +import com.google.common.io.Closeables; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.routing.*; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.compress.lzf.LZF; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.io.stream.BytesStreamInput; +import org.elasticsearch.common.io.stream.CachedStreamInput; +import org.elasticsearch.common.io.stream.CachedStreamOutput; +import org.elasticsearch.common.io.stream.LZFStreamInput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.index.shard.ShardId; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +/** + */ +public class LocalGatewayShardsState extends AbstractComponent implements ClusterStateListener { + + private final NodeEnvironment nodeEnv; + private final ClusterService clusterService; + + private volatile boolean initialized = false; + private volatile Map currentState = Maps.newHashMap(); + + @Inject + public LocalGatewayShardsState(Settings settings, NodeEnvironment nodeEnv, ClusterService clusterService, TransportNodesListGatewayStartedShards listGatewayStartedShards) { + super(settings); + this.nodeEnv = nodeEnv; + this.clusterService = clusterService; + listGatewayStartedShards.initGateway(this); + } + + public Map currentStartedShards() { + lazyInitialize(); + return this.currentState; + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (event.state().blocks().disableStatePersistence()) { + return; + } + + if (!event.state().nodes().localNode().dataNode()) { + return; + } + + if (!event.routingTableChanged()) { + return; + } + + Map newState = Maps.newHashMap(); + newState.putAll(this.currentState); + + + // remove from the current state all the shards that are completely started somewhere, we won't need them anymore + // and if they are still here, we will add them in the next phase + // Also note, this works well when closing an index, since a closed index will have no routing shards entries + // so they won't get removed (we want to keep the fact that those shards are allocated on this node if needed) + for (IndexRoutingTable indexRoutingTable : event.state().routingTable()) { + for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { + if (indexShardRoutingTable.countWithState(ShardRoutingState.STARTED) == indexShardRoutingTable.size()) { + newState.remove(indexShardRoutingTable.shardId()); + } + } + } + // remove deleted indices from the started shards + for (ShardId shardId : currentState.keySet()) { + if (!event.state().metaData().hasIndex(shardId.index().name())) { + newState.remove(shardId); + } + } + // now, add all the ones that are active and on this node + RoutingNode routingNode = event.state().readOnlyRoutingNodes().node(event.state().nodes().localNodeId()); + if (routingNode != null) { + // our node is not in play yet... + for (MutableShardRouting shardRouting : routingNode) { + if (shardRouting.active()) { + newState.put(shardRouting.shardId(), new ShardStateInfo(shardRouting.version())); + } + } + } + + // go over the write started shards if needed + for (Iterator> it = newState.entrySet().iterator(); it.hasNext(); ) { + Map.Entry entry = it.next(); + ShardId shardId = entry.getKey(); + ShardStateInfo shardStateInfo = entry.getValue(); + + String writeReason = null; + ShardStateInfo currentShardStateInfo = currentState.get(shardId); + if (currentShardStateInfo == null) { + writeReason = "freshly started, version [" + shardStateInfo.version + "]"; + } else if (currentShardStateInfo.version != shardStateInfo.version) { + writeReason = "version changed from [" + currentShardStateInfo.version + "] to [" + shardStateInfo.version + "]"; + } + + // we update the write reason if we really need to write a new one... + if (writeReason == null) { + continue; + } + + try { + writeShardState(writeReason, shardId, shardStateInfo, currentShardStateInfo); + } catch (Exception e) { + // we failed to write the shard state, remove it from our builder, we will try and write + // it next time... + it.remove(); + } + } + + // now, go over the current ones and delete ones that are not in the new one + for (Map.Entry entry : currentState.entrySet()) { + ShardId shardId = entry.getKey(); + if (!newState.containsKey(shardId)) { + deleteShardState(shardId); + } + } + + this.currentState = newState; + } + + private synchronized void lazyInitialize() { + if (initialized) { + return; + } + initialized = true; + + // we only persist shards state for data nodes + if (!clusterService.localNode().dataNode()) { + return; + } + + try { + pre019Upgrade(); + long start = System.currentTimeMillis(); + loadStartedShards(); + logger.debug("took {} to load started shards state", TimeValue.timeValueMillis(System.currentTimeMillis() - start)); + } catch (Exception e) { + logger.error("failed to read local state (started shards), exiting...", e); + // ugly, but, if we fail to read it, bail completely so we don't have any node corrupting the cluster + System.exit(1); + } + } + + + private void loadStartedShards() throws Exception { + Set shardIds = nodeEnv.findAllShardIds(); + long highestVersion = -1; + Map shardsState = Maps.newHashMap(); + for (ShardId shardId : shardIds) { + long highestShardVersion = -1; + File highestShardFile = null; + for (File shardLocation : nodeEnv.shardLocations(shardId)) { + File shardStateDir = new File(shardLocation, "_state"); + if (!shardStateDir.exists() || !shardStateDir.isDirectory()) { + continue; + } + // now, iterate over the current versions, and find latest one + File[] stateFiles = shardStateDir.listFiles(); + if (stateFiles == null) { + continue; + } + for (File stateFile : stateFiles) { + if (!stateFile.getName().startsWith("state-")) { + continue; + } + try { + long version = Long.parseLong(stateFile.getName().substring("state-".length())); + if (version > highestShardVersion) { + byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile)); + if (data.length == 0) { + logger.debug("[{}][{}]: not data for [" + stateFile.getAbsolutePath() + "], ignoring...", shardId.index().name(), shardId.id()); + continue; + } + long readStateVersion = readShardState(data); + assert readStateVersion == version; + highestShardVersion = version; + highestShardFile = stateFile; + } + } catch (Exception e) { + logger.debug("[{}][{}]: failed to read [" + stateFile.getAbsolutePath() + "], ignoring...", e, shardId.index().name(), shardId.id()); + } + } + } + // did we find a state file? + if (highestShardFile == null) { + continue; + } + + shardsState.put(shardId, new ShardStateInfo(highestShardVersion)); + + // update the global version + if (highestShardVersion > highestVersion) { + highestVersion = highestShardVersion; + } + } + // update the current started shards only if there is data there... + if (highestVersion != -1) { + currentState = shardsState; + } + } + + private long readShardState(byte[] data) throws Exception { + XContentParser parser = null; + try { + if (LZF.isCompressed(data)) { + BytesStreamInput siBytes = new BytesStreamInput(data, false); + LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes); + parser = XContentFactory.xContent(XContentType.JSON).createParser(siLzf); + } else { + parser = XContentFactory.xContent(XContentType.JSON).createParser(data); + } + XContentParser.Token token = parser.nextToken(); + if (token == null) { + return -1; + } + long version = -1; + String currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if ("version".equals(currentFieldName)) { + version = parser.longValue(); + } + } + } + return version; + } finally { + if (parser != null) { + parser.close(); + } + } + } + + private void writeShardState(String reason, ShardId shardId, ShardStateInfo shardStateInfo, @Nullable ShardStateInfo previousStateInfo) throws Exception { + logger.trace("[{}][{}] writing shard state, reason [{}]", shardId.index().name(), shardId.id(), reason); + CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry(); + try { + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, cachedEntry.cachedBytes()); + builder.prettyPrint(); + builder.startObject(); + builder.field("version", shardStateInfo.version); + builder.endObject(); + builder.flush(); + + Exception lastFailure = null; + boolean wroteAtLeastOnce = false; + for (File shardLocation : nodeEnv.shardLocations(shardId)) { + File shardStateDir = new File(shardLocation, "_state"); + FileSystemUtils.mkdirs(shardStateDir); + File stateFile = new File(shardStateDir, "state-" + shardStateInfo.version); + + + FileOutputStream fos = null; + try { + fos = new FileOutputStream(stateFile); + fos.write(cachedEntry.bytes().underlyingBytes(), 0, cachedEntry.bytes().size()); + fos.getChannel().force(true); + Closeables.closeQuietly(fos); + wroteAtLeastOnce = true; + } catch (Exception e) { + lastFailure = e; + } finally { + Closeables.closeQuietly(fos); + } + } + + if (!wroteAtLeastOnce) { + logger.warn("[{}][{}]: failed to write shard state", shardId.index().name(), shardId.id(), lastFailure); + throw new IOException("failed to write shard state for " + shardId, lastFailure); + } + + // delete the old files + if (previousStateInfo != null) { + for (File shardLocation : nodeEnv.shardLocations(shardId)) { + File stateFile = new File(new File(shardLocation, "_state"), "state-" + previousStateInfo.version); + stateFile.delete(); + } + } + } finally { + CachedStreamOutput.pushEntry(cachedEntry); + } + } + + private void deleteShardState(ShardId shardId) { + logger.trace("[{}][{}] delete shard state", shardId.index().name(), shardId.id()); + File[] shardLocations = nodeEnv.shardLocations(shardId); + for (File shardLocation : shardLocations) { + if (!shardLocation.exists()) { + continue; + } + FileSystemUtils.deleteRecursively(new File(shardLocation, "_state")); + } + } + + private void pre019Upgrade() throws Exception { + long index = -1; + File latest = null; + for (File dataLocation : nodeEnv.nodeDataLocations()) { + File stateLocation = new File(dataLocation, "_state"); + if (!stateLocation.exists()) { + continue; + } + File[] stateFiles = stateLocation.listFiles(); + if (stateFiles == null) { + continue; + } + for (File stateFile : stateFiles) { + if (logger.isTraceEnabled()) { + logger.trace("[find_latest_state]: processing [" + stateFile.getName() + "]"); + } + String name = stateFile.getName(); + if (!name.startsWith("shards-")) { + continue; + } + long fileIndex = Long.parseLong(name.substring(name.indexOf('-') + 1)); + if (fileIndex >= index) { + // try and read the meta data + try { + byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile)); + if (data.length == 0) { + logger.debug("[find_latest_state]: not data for [" + name + "], ignoring..."); + } + pre09ReadState(data); + index = fileIndex; + latest = stateFile; + } catch (IOException e) { + logger.warn("[find_latest_state]: failed to read state from [" + name + "], ignoring...", e); + } + } + } + } + if (latest == null) { + return; + } + + logger.info("found old shards state, loading started shards from [{}] and converting to new shards state locations...", latest.getAbsolutePath()); + Map shardsState = pre09ReadState(Streams.copyToByteArray(new FileInputStream(latest))); + + for (Map.Entry entry : shardsState.entrySet()) { + writeShardState("upgrade", entry.getKey(), entry.getValue(), null); + } + + // rename shards state to backup state + File backupFile = new File(latest.getParentFile(), "backup-" + latest.getName()); + if (!latest.renameTo(backupFile)) { + throw new IOException("failed to rename old state to backup state [" + latest.getAbsolutePath() + "]"); + } + + // delete all other shards state files + for (File dataLocation : nodeEnv.nodeDataLocations()) { + File stateLocation = new File(dataLocation, "_state"); + if (!stateLocation.exists()) { + continue; + } + File[] stateFiles = stateLocation.listFiles(); + if (stateFiles == null) { + continue; + } + for (File stateFile : stateFiles) { + if (logger.isTraceEnabled()) { + logger.trace("[find_latest_state]: processing [" + stateFile.getName() + "]"); + } + String name = stateFile.getName(); + if (!name.startsWith("shards-")) { + continue; + } + stateFile.delete(); + } + } + + logger.info("conversion to new shards state location and format done, backup create at [{}]", backupFile.getAbsolutePath()); + } + + private Map pre09ReadState(byte[] data) throws IOException { + XContentParser parser = null; + try { + Map shardsState = Maps.newHashMap(); + + if (LZF.isCompressed(data)) { + BytesStreamInput siBytes = new BytesStreamInput(data, false); + LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes); + parser = XContentFactory.xContent(XContentType.JSON).createParser(siLzf); + } else { + parser = XContentFactory.xContent(XContentType.JSON).createParser(data); + } + + String currentFieldName = null; + XContentParser.Token token = parser.nextToken(); + if (token == null) { + // no data... + return shardsState; + } + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_ARRAY) { + if ("shards".equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.START_OBJECT) { + String shardIndex = null; + int shardId = -1; + long version = -1; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if ("index".equals(currentFieldName)) { + shardIndex = parser.text(); + } else if ("id".equals(currentFieldName)) { + shardId = parser.intValue(); + } else if ("version".equals(currentFieldName)) { + version = parser.longValue(); + } + } + } + shardsState.put(new ShardId(shardIndex, shardId), new ShardStateInfo(version)); + } + } + } + } + } + return shardsState; + } finally { + if (parser != null) { + parser.close(); + } + } + } +} diff --git a/src/main/java/org/elasticsearch/gateway/local/state/shards/ShardStateInfo.java b/src/main/java/org/elasticsearch/gateway/local/state/shards/ShardStateInfo.java new file mode 100644 index 00000000000..764ba936ad2 --- /dev/null +++ b/src/main/java/org/elasticsearch/gateway/local/state/shards/ShardStateInfo.java @@ -0,0 +1,31 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gateway.local.state.shards; + +/** + */ +public class ShardStateInfo { + + public final long version; + + public ShardStateInfo(long version) { + this.version = version; + } +} diff --git a/src/main/java/org/elasticsearch/gateway/local/TransportNodesListGatewayStartedShards.java b/src/main/java/org/elasticsearch/gateway/local/state/shards/TransportNodesListGatewayStartedShards.java similarity index 94% rename from src/main/java/org/elasticsearch/gateway/local/TransportNodesListGatewayStartedShards.java rename to src/main/java/org/elasticsearch/gateway/local/state/shards/TransportNodesListGatewayStartedShards.java index 24e86aba66b..b0cca8da8a6 100644 --- a/src/main/java/org/elasticsearch/gateway/local/TransportNodesListGatewayStartedShards.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/shards/TransportNodesListGatewayStartedShards.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.gateway.local; +package org.elasticsearch.gateway.local.state.shards; import com.google.common.collect.Lists; import org.elasticsearch.ElasticSearchException; @@ -48,15 +48,15 @@ import java.util.concurrent.atomic.AtomicReferenceArray; */ public class TransportNodesListGatewayStartedShards extends TransportNodesOperationAction { - private LocalGateway gateway; + private LocalGatewayShardsState shardsState; @Inject public TransportNodesListGatewayStartedShards(Settings settings, ClusterName clusterName, ThreadPool threadPool, ClusterService clusterService, TransportService transportService) { super(settings, clusterName, threadPool, clusterService, transportService); } - TransportNodesListGatewayStartedShards initGateway(LocalGateway gateway) { - this.gateway = gateway; + TransportNodesListGatewayStartedShards initGateway(LocalGatewayShardsState shardsState) { + this.shardsState = shardsState; return this; } @@ -117,12 +117,11 @@ public class TransportNodesListGatewayStartedShards extends TransportNodesOperat @Override protected NodeLocalGatewayStartedShards nodeOperation(NodeRequest request) throws ElasticSearchException { - LocalGatewayStartedShards startedShards = gateway.currentStartedShards(); - if (startedShards != null) { - for (Map.Entry entry : startedShards.shards().entrySet()) { + Map shardsStateInfo = shardsState.currentStartedShards(); + if (shardsStateInfo != null) { + for (Map.Entry entry : shardsStateInfo.entrySet()) { if (entry.getKey().equals(request.shardId)) { - assert entry.getValue() != null; - return new NodeLocalGatewayStartedShards(clusterService.localNode(), entry.getValue()); + return new NodeLocalGatewayStartedShards(clusterService.localNode(), entry.getValue().version); } } } diff --git a/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java b/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java index fcd3124eeff..557a2d784f2 100644 --- a/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java +++ b/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java @@ -254,8 +254,8 @@ public class SimpleRecoveryLocalGatewayTests extends AbstractNodesTests { buildNode("node2", settingsBuilder().put("gateway.type", "local").build()); cleanAndCloseNodes(); - Node node1 = startNode("node1", settingsBuilder().put("gateway.type", "local").put("index.number_of_shards", 1).build()); - Node node2 = startNode("node2", settingsBuilder().put("gateway.type", "local").put("index.number_of_shards", 1).build()); + Node node1 = startNode("node1", settingsBuilder().put("gateway.type", "local").put("index.number_of_shards", 1).put("gateway.recover_after_nodes", 2).build()); + Node node2 = startNode("node2", settingsBuilder().put("gateway.type", "local").put("index.number_of_shards", 1).put("gateway.recover_after_nodes", 2).build()); node1.client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).execute().actionGet(); node1.client().admin().indices().prepareFlush().execute().actionGet(); From 64358948eff4e3b75134dc9e2776e9dc5a39e156 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 18 Jan 2012 15:08:37 +0200 Subject: [PATCH 182/270] Failure exception while executing a valid query after an invalid query, closes #1617. --- .../search/type/TransportSearchCache.java | 16 ---------- ...TransportSearchDfsQueryAndFetchAction.java | 2 +- ...ransportSearchDfsQueryThenFetchAction.java | 4 +-- .../search/type/TransportSearchHelper.java | 15 ---------- .../TransportSearchQueryThenFetchAction.java | 2 +- ...nsportSearchScrollQueryAndFetchAction.java | 28 +++++++++++++---- ...sportSearchScrollQueryThenFetchAction.java | 28 +++++++++++++---- .../type/TransportSearchScrollScanAction.java | 30 ++++++++++++++----- .../type/TransportSearchTypeAction.java | 30 ++++++++++++------- 9 files changed, 91 insertions(+), 64 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchCache.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchCache.java index 631bac8014f..2c238edfcbe 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchCache.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchCache.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.search.type; import jsr166y.LinkedTransferQueue; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.dfs.DfsSearchResult; @@ -37,8 +36,6 @@ import java.util.Queue; */ public class TransportSearchCache { - private final Queue> cacheShardFailures = new LinkedTransferQueue>(); - private final Queue> cacheDfsResults = new LinkedTransferQueue>(); private final Queue> cacheQueryResults = new LinkedTransferQueue>(); @@ -48,19 +45,6 @@ public class TransportSearchCache { private final Queue> cacheQueryFetchResults = new LinkedTransferQueue>(); - public Collection obtainShardFailures() { - Collection shardFailures; - while ((shardFailures = cacheShardFailures.poll()) == null) { - cacheShardFailures.offer(new LinkedTransferQueue()); - } - return shardFailures; - } - - public void releaseShardFailures(Collection shardFailures) { - shardFailures.clear(); - cacheShardFailures.offer(shardFailures); - } - public Collection obtainDfsResults() { Collection dfsSearchResults; while ((dfsSearchResults = cacheDfsResults.poll()) == null) { diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryAndFetchAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryAndFetchAction.java index 20bc390d890..3f2aa17e33f 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryAndFetchAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryAndFetchAction.java @@ -153,7 +153,7 @@ public class TransportSearchDfsQueryAndFetchAction extends TransportSearchTypeAc if (logger.isDebugEnabled()) { logger.debug("[{}] Failed to execute query phase", t, querySearchRequest.id()); } - AsyncAction.this.shardFailures.add(new ShardSearchFailure(t)); + AsyncAction.this.addShardFailure(new ShardSearchFailure(t)); successulOps.decrementAndGet(); if (counter.decrementAndGet() == 0) { finishHim(); diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryThenFetchAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryThenFetchAction.java index 4eb0ecb0a88..53a0aee72c9 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryThenFetchAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryThenFetchAction.java @@ -160,7 +160,7 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA if (logger.isDebugEnabled()) { logger.debug("[{}] Failed to execute query phase", t, querySearchRequest.id()); } - AsyncAction.this.shardFailures.add(new ShardSearchFailure(t)); + AsyncAction.this.addShardFailure(new ShardSearchFailure(t)); successulOps.decrementAndGet(); if (counter.decrementAndGet() == 0) { executeFetchPhase(); @@ -251,7 +251,7 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA if (logger.isDebugEnabled()) { logger.debug("[{}] Failed to execute fetch phase", t, fetchSearchRequest.id()); } - AsyncAction.this.shardFailures.add(new ShardSearchFailure(t)); + AsyncAction.this.addShardFailure(new ShardSearchFailure(t)); successulOps.decrementAndGet(); if (counter.decrementAndGet() == 0) { finishHim(); diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java index c7425f1942b..1baa5aed7ba 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java @@ -26,7 +26,6 @@ import org.elasticsearch.ElasticSearchIllegalStateException; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Base64; import org.elasticsearch.common.Nullable; @@ -47,20 +46,6 @@ import java.util.Map; public abstract class TransportSearchHelper { - /** - * Builds the shard failures, and releases the cache (meaning this should only be called once!). - */ - public static ShardSearchFailure[] buildShardFailures(Collection shardFailures, TransportSearchCache searchCache) { - ShardSearchFailure[] ret; - if (shardFailures.isEmpty()) { - ret = ShardSearchFailure.EMPTY_ARRAY; - } else { - ret = shardFailures.toArray(ShardSearchFailure.EMPTY_ARRAY); - } - searchCache.releaseShardFailures(shardFailures); - return ret; - } - public static InternalSearchRequest internalSearchRequest(ShardRouting shardRouting, int numberOfShards, SearchRequest request, String[] filteringAliases, long nowInMillis) { InternalSearchRequest internalRequest = new InternalSearchRequest(shardRouting, numberOfShards, request.searchType()); internalRequest.source(request.source(), request.sourceOffset(), request.sourceLength()); diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchQueryThenFetchAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchQueryThenFetchAction.java index 6ba16d447f1..b4ef8124a8a 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchQueryThenFetchAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchQueryThenFetchAction.java @@ -161,7 +161,7 @@ public class TransportSearchQueryThenFetchAction extends TransportSearchTypeActi if (logger.isDebugEnabled()) { logger.debug("[{}] Failed to execute fetch phase", t, fetchSearchRequest.id()); } - AsyncAction.this.shardFailures.add(new ShardSearchFailure(t)); + AsyncAction.this.addShardFailure(new ShardSearchFailure(t)); successulOps.decrementAndGet(); if (counter.decrementAndGet() == 0) { finishHim(); diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryAndFetchAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryAndFetchAction.java index 8ee85f779d3..b5e6afcff35 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryAndFetchAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryAndFetchAction.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.search.type; +import jsr166y.LinkedTransferQueue; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.*; import org.elasticsearch.cluster.ClusterService; @@ -37,11 +38,9 @@ import org.elasticsearch.search.fetch.QueryFetchSearchResult; import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.threadpool.ThreadPool; -import java.util.Collection; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import static org.elasticsearch.action.search.type.TransportSearchHelper.buildShardFailures; import static org.elasticsearch.action.search.type.TransportSearchHelper.internalScrollSearchRequest; /** @@ -85,7 +84,7 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent private final DiscoveryNodes nodes; - protected final Collection shardFailures = searchCache.obtainShardFailures(); + private volatile LinkedTransferQueue shardFailures; private final Map queryFetchResults = searchCache.obtainQueryFetchResults(); @@ -104,6 +103,23 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent this.counter = new AtomicInteger(scrollId.context().length); } + protected final ShardSearchFailure[] buildShardFailures() { + LinkedTransferQueue localFailures = shardFailures; + if (localFailures == null) { + return ShardSearchFailure.EMPTY_ARRAY; + } + return localFailures.toArray(ShardSearchFailure.EMPTY_ARRAY); + } + + // we do our best to return the shard failures, but its ok if its not fully concurrently safe + // we simply try and return as much as possible + protected final void addShardFailure(ShardSearchFailure failure) { + if (shardFailures == null) { + shardFailures = new LinkedTransferQueue(); + } + shardFailures.add(failure); + } + public void start() { if (scrollId.context().length == 0) { listener.onFailure(new SearchPhaseExecutionException("query", "no nodes to search on", null)); @@ -193,7 +209,7 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent if (logger.isDebugEnabled()) { logger.debug("[{}] Failed to execute query phase", t, searchId); } - shardFailures.add(new ShardSearchFailure(t)); + addShardFailure(new ShardSearchFailure(t)); successfulOps.decrementAndGet(); if (counter.decrementAndGet() == 0) { finishHim(); @@ -206,7 +222,7 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent try { innerFinishHim(); } catch (Exception e) { - listener.onFailure(new ReduceSearchPhaseException("fetch", "", e, buildShardFailures(shardFailures, searchCache))); + listener.onFailure(new ReduceSearchPhaseException("fetch", "", e, buildShardFailures())); } } @@ -219,7 +235,7 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent } searchCache.releaseQueryFetchResults(queryFetchResults); listener.onResponse(new SearchResponse(internalResponse, scrollId, this.scrollId.context().length, successfulOps.get(), - System.currentTimeMillis() - startTime, buildShardFailures(shardFailures, searchCache))); + System.currentTimeMillis() - startTime, buildShardFailures())); } } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java index 928a737f89f..4eabc61af75 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.search.type; +import jsr166y.LinkedTransferQueue; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.*; import org.elasticsearch.cluster.ClusterService; @@ -41,11 +42,9 @@ import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.query.QuerySearchResultProvider; import org.elasticsearch.threadpool.ThreadPool; -import java.util.Collection; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import static org.elasticsearch.action.search.type.TransportSearchHelper.buildShardFailures; import static org.elasticsearch.action.search.type.TransportSearchHelper.internalScrollSearchRequest; /** @@ -89,7 +88,7 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent private final DiscoveryNodes nodes; - protected final Collection shardFailures = searchCache.obtainShardFailures(); + protected volatile LinkedTransferQueue shardFailures; private final Map queryResults = searchCache.obtainQueryResults(); @@ -109,6 +108,23 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent this.successfulOps = new AtomicInteger(scrollId.context().length); } + protected final ShardSearchFailure[] buildShardFailures() { + LinkedTransferQueue localFailures = shardFailures; + if (localFailures == null) { + return ShardSearchFailure.EMPTY_ARRAY; + } + return localFailures.toArray(ShardSearchFailure.EMPTY_ARRAY); + } + + // we do our best to return the shard failures, but its ok if its not fully concurrently safe + // we simply try and return as much as possible + protected final void addShardFailure(ShardSearchFailure failure) { + if (shardFailures == null) { + shardFailures = new LinkedTransferQueue(); + } + shardFailures.add(failure); + } + public void start() { if (scrollId.context().length == 0) { listener.onFailure(new SearchPhaseExecutionException("query", "no nodes to search on", null)); @@ -185,7 +201,7 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent if (logger.isDebugEnabled()) { logger.debug("[{}] Failed to execute query phase", t, searchId); } - shardFailures.add(new ShardSearchFailure(t)); + addShardFailure(new ShardSearchFailure(t)); successfulOps.decrementAndGet(); if (counter.decrementAndGet() == 0) { executeFetchPhase(); @@ -237,7 +253,7 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent try { innerFinishHim(); } catch (Exception e) { - listener.onFailure(new ReduceSearchPhaseException("fetch", "", e, buildShardFailures(shardFailures, searchCache))); + listener.onFailure(new ReduceSearchPhaseException("fetch", "", e, buildShardFailures())); } } @@ -248,7 +264,7 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent scrollId = request.scrollId(); } listener.onResponse(new SearchResponse(internalResponse, scrollId, this.scrollId.context().length, successfulOps.get(), - System.currentTimeMillis() - startTime, buildShardFailures(shardFailures, searchCache))); + System.currentTimeMillis() - startTime, buildShardFailures())); searchCache.releaseQueryResults(queryResults); searchCache.releaseFetchResults(fetchResults); } diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollScanAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollScanAction.java index 4b9bfa1be0a..b417573ced9 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollScanAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollScanAction.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.search.type; +import jsr166y.LinkedTransferQueue; import org.apache.lucene.search.ScoreDoc; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.*; @@ -41,11 +42,9 @@ import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; -import java.util.Collection; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import static org.elasticsearch.action.search.type.TransportSearchHelper.buildShardFailures; import static org.elasticsearch.action.search.type.TransportSearchHelper.internalScrollSearchRequest; /** @@ -89,7 +88,7 @@ public class TransportSearchScrollScanAction extends AbstractComponent { private final DiscoveryNodes nodes; - protected final Collection shardFailures = searchCache.obtainShardFailures(); + protected volatile LinkedTransferQueue shardFailures; private final Map queryFetchResults = searchCache.obtainQueryFetchResults(); @@ -108,11 +107,28 @@ public class TransportSearchScrollScanAction extends AbstractComponent { this.counter = new AtomicInteger(scrollId.context().length); } + protected final ShardSearchFailure[] buildShardFailures() { + LinkedTransferQueue localFailures = shardFailures; + if (localFailures == null) { + return ShardSearchFailure.EMPTY_ARRAY; + } + return localFailures.toArray(ShardSearchFailure.EMPTY_ARRAY); + } + + // we do our best to return the shard failures, but its ok if its not fully concurrently safe + // we simply try and return as much as possible + protected final void addShardFailure(ShardSearchFailure failure) { + if (shardFailures == null) { + shardFailures = new LinkedTransferQueue(); + } + shardFailures.add(failure); + } + public void start() { if (scrollId.context().length == 0) { final InternalSearchResponse internalResponse = new InternalSearchResponse(new InternalSearchHits(InternalSearchHits.EMPTY, Long.parseLong(this.scrollId.attributes().get("total_hits")), 0.0f), null, false); searchCache.releaseQueryFetchResults(queryFetchResults); - listener.onResponse(new SearchResponse(internalResponse, request.scrollId(), 0, 0, 0l, TransportSearchHelper.buildShardFailures(shardFailures, searchCache))); + listener.onResponse(new SearchResponse(internalResponse, request.scrollId(), 0, 0, 0l, buildShardFailures())); return; } @@ -199,7 +215,7 @@ public class TransportSearchScrollScanAction extends AbstractComponent { if (logger.isDebugEnabled()) { logger.debug("[{}] Failed to execute query phase", t, searchId); } - shardFailures.add(new ShardSearchFailure(t)); + addShardFailure(new ShardSearchFailure(t)); successfulOps.decrementAndGet(); if (counter.decrementAndGet() == 0) { finishHim(); @@ -212,7 +228,7 @@ public class TransportSearchScrollScanAction extends AbstractComponent { try { innerFinishHim(); } catch (Exception e) { - ReduceSearchPhaseException failure = new ReduceSearchPhaseException("fetch", "", e, buildShardFailures(shardFailures, searchCache)); + ReduceSearchPhaseException failure = new ReduceSearchPhaseException("fetch", "", e, buildShardFailures()); if (logger.isDebugEnabled()) { logger.debug("failed to reduce search", failure); } @@ -252,7 +268,7 @@ public class TransportSearchScrollScanAction extends AbstractComponent { scrollId = TransportSearchHelper.buildScrollId(this.scrollId.type(), queryFetchResults.values(), this.scrollId.attributes()); // continue moving the total_hits } listener.onResponse(new SearchResponse(internalResponse, scrollId, this.scrollId.context().length, successfulOps.get(), - System.currentTimeMillis() - startTime, buildShardFailures(shardFailures, searchCache))); + System.currentTimeMillis() - startTime, buildShardFailures())); } } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java index 7ab9f545640..53f77710399 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.search.type; +import jsr166y.LinkedTransferQueue; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.*; import org.elasticsearch.action.support.TransportAction; @@ -44,7 +45,6 @@ import org.elasticsearch.search.query.QuerySearchResultProvider; import org.elasticsearch.threadpool.ThreadPool; import java.util.Arrays; -import java.util.Collection; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -92,7 +92,7 @@ public abstract class TransportSearchTypeAction extends TransportAction shardFailures = searchCache.obtainShardFailures(); + private volatile LinkedTransferQueue shardFailures; protected volatile ShardDoc[] sortedShardList; @@ -248,9 +248,9 @@ public abstract class TransportSearchTypeAction extends TransportAction localFailures = shardFailures; + if (localFailures == null) { + return ShardSearchFailure.EMPTY_ARRAY; + } + return localFailures.toArray(ShardSearchFailure.EMPTY_ARRAY); + } + + // we do our best to return the shard failures, but its ok if its not fully concurrently safe + // we simply try and return as much as possible + protected final void addShardFailure(ShardSearchFailure failure) { + if (shardFailures == null) { + shardFailures = new LinkedTransferQueue(); + } + shardFailures.add(failure); } /** From 0a3c941947bf95af24eeaf58a5b40d7b5d8e76bf Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 18 Jan 2012 21:00:09 +0200 Subject: [PATCH 183/270] Node Stats: Add fs level stats (size + iostats), closes #1622. --- .../admin/cluster/node/stats/NodeStats.java | 32 +- .../cluster/node/stats/NodesStatsRequest.java | 17 ++ .../node/stats/NodesStatsRequestBuilder.java | 8 + .../node/stats/NodesStatsResponse.java | 3 + .../node/stats/TransportNodesStatsAction.java | 2 +- .../common/io/stream/AdapterStreamInput.java | 5 + .../common/io/stream/AdapterStreamOutput.java | 5 + .../common/io/stream/StreamInput.java | 8 + .../common/io/stream/StreamOutput.java | 9 + .../elasticsearch/monitor/MonitorModule.java | 7 + .../elasticsearch/monitor/MonitorService.java | 11 +- .../org/elasticsearch/monitor/fs/FsProbe.java | 27 ++ .../elasticsearch/monitor/fs/FsService.java | 55 ++++ .../org/elasticsearch/monitor/fs/FsStats.java | 284 ++++++++++++++++++ .../elasticsearch/monitor/fs/JmxFsProbe.java | 59 ++++ .../monitor/fs/SigarFsProbe.java | 94 ++++++ .../node/service/NodeService.java | 4 +- .../node/stats/RestNodesStatsAction.java | 13 + 18 files changed, 639 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/elasticsearch/monitor/fs/FsProbe.java create mode 100644 src/main/java/org/elasticsearch/monitor/fs/FsService.java create mode 100644 src/main/java/org/elasticsearch/monitor/fs/FsStats.java create mode 100644 src/main/java/org/elasticsearch/monitor/fs/JmxFsProbe.java create mode 100644 src/main/java/org/elasticsearch/monitor/fs/SigarFsProbe.java diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java index 76f55a94c5d..56858d1a6e2 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.http.HttpStats; import org.elasticsearch.indices.NodeIndicesStats; +import org.elasticsearch.monitor.fs.FsStats; import org.elasticsearch.monitor.jvm.JvmStats; import org.elasticsearch.monitor.network.NetworkStats; import org.elasticsearch.monitor.os.OsStats; @@ -61,6 +62,9 @@ public class NodeStats extends NodeOperationResponse { @Nullable private NetworkStats network; + @Nullable + private FsStats fs; + @Nullable private TransportStats transport; @@ -72,7 +76,7 @@ public class NodeStats extends NodeOperationResponse { public NodeStats(DiscoveryNode node, @Nullable String hostname, @Nullable NodeIndicesStats indices, @Nullable OsStats os, @Nullable ProcessStats process, @Nullable JvmStats jvm, @Nullable ThreadPoolStats threadPool, @Nullable NetworkStats network, - @Nullable TransportStats transport, @Nullable HttpStats http) { + @Nullable FsStats fs, @Nullable TransportStats transport, @Nullable HttpStats http) { super(node); this.hostname = hostname; this.indices = indices; @@ -81,6 +85,7 @@ public class NodeStats extends NodeOperationResponse { this.jvm = jvm; this.threadPool = threadPool; this.network = network; + this.fs = fs; this.transport = transport; this.http = http; } @@ -191,6 +196,22 @@ public class NodeStats extends NodeOperationResponse { return network(); } + /** + * File system level stats. + */ + @Nullable + FsStats fs() { + return fs; + } + + /** + * File system level stats. + */ + @Nullable + FsStats getFs() { + return fs(); + } + @Nullable public TransportStats transport() { return this.transport; @@ -241,6 +262,9 @@ public class NodeStats extends NodeOperationResponse { if (in.readBoolean()) { network = NetworkStats.readNetworkStats(in); } + if (in.readBoolean()) { + fs = FsStats.readFsStats(in); + } if (in.readBoolean()) { transport = TransportStats.readTransportStats(in); } @@ -294,6 +318,12 @@ public class NodeStats extends NodeOperationResponse { out.writeBoolean(true); network.writeTo(out); } + if (fs == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + fs.writeTo(out); + } if (transport == null) { out.writeBoolean(false); } else { diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java index c8a3519262e..0e9f0648e93 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -36,6 +36,7 @@ public class NodesStatsRequest extends NodesOperationRequest { private boolean jvm; private boolean threadPool; private boolean network; + private boolean fs; private boolean transport; private boolean http; @@ -60,6 +61,7 @@ public class NodesStatsRequest extends NodesOperationRequest { this.jvm = false; this.threadPool = false; this.network = false; + this.fs = false; this.transport = false; this.http = false; return this; @@ -155,6 +157,21 @@ public class NodesStatsRequest extends NodesOperationRequest { return this; } + /** + * Should the node file system stats be returned. + */ + public boolean fs() { + return this.fs; + } + + /** + * Should the node file system stats be returned. + */ + public NodesStatsRequest fs(boolean fs) { + this.fs = fs; + return this; + } + /** * Should the node Transport be returned. */ diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java index d5724bc7947..dbca5dfa3a0 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java @@ -93,6 +93,14 @@ public class NodesStatsRequestBuilder extends BaseClusterRequestBuilder implem if (nodeStats.network() != null) { nodeStats.network().toXContent(builder, params); } + if (nodeStats.fs() != null) { + nodeStats.fs().toXContent(builder, params); + } if (nodeStats.transport() != null) { nodeStats.transport().toXContent(builder, params); } diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index 8019a47331f..1ad7aa966d3 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -97,7 +97,7 @@ public class TransportNodesStatsAction extends TransportNodesOperationAction { private final NetworkService networkService; + private final FsService fsService; + @Inject public MonitorService(Settings settings, JvmMonitorService jvmMonitorService, - OsService osService, ProcessService processService, JvmService jvmService, NetworkService networkService) { + OsService osService, ProcessService processService, JvmService jvmService, NetworkService networkService, + FsService fsService) { super(settings); this.jvmMonitorService = jvmMonitorService; this.osService = osService; this.processService = processService; this.jvmService = jvmService; this.networkService = networkService; + this.fsService = fsService; } public OsService osService() { @@ -71,6 +76,10 @@ public class MonitorService extends AbstractLifecycleComponent { return this.networkService; } + public FsService fsService() { + return this.fsService; + } + @Override protected void doStart() throws ElasticSearchException { jvmMonitorService.start(); diff --git a/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java b/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java new file mode 100644 index 00000000000..609b22512e2 --- /dev/null +++ b/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java @@ -0,0 +1,27 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.monitor.fs; + +/** + */ +public interface FsProbe { + + FsStats stats(); +} diff --git a/src/main/java/org/elasticsearch/monitor/fs/FsService.java b/src/main/java/org/elasticsearch/monitor/fs/FsService.java new file mode 100644 index 00000000000..ce77f8a5778 --- /dev/null +++ b/src/main/java/org/elasticsearch/monitor/fs/FsService.java @@ -0,0 +1,55 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.monitor.fs; + +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; + +/** + */ +public class FsService extends AbstractComponent { + + private final FsProbe probe; + + private final TimeValue refreshInterval; + + private FsStats cachedStats; + + @Inject + public FsService(Settings settings, FsProbe probe) { + super(settings); + this.probe = probe; + this.cachedStats = probe.stats(); + + this.refreshInterval = componentSettings.getAsTime("refresh_interval", TimeValue.timeValueSeconds(1)); + + logger.debug("Using probe [{}] with refresh_interval [{}]", probe, refreshInterval); + } + + public synchronized FsStats stats() { + if ((System.currentTimeMillis() - cachedStats.timestamp()) > refreshInterval.millis()) { + cachedStats = probe.stats(); + } + return cachedStats; + } + +} diff --git a/src/main/java/org/elasticsearch/monitor/fs/FsStats.java b/src/main/java/org/elasticsearch/monitor/fs/FsStats.java new file mode 100644 index 00000000000..f18749177c1 --- /dev/null +++ b/src/main/java/org/elasticsearch/monitor/fs/FsStats.java @@ -0,0 +1,284 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.monitor.fs; + +import com.google.common.collect.Iterators; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Iterator; + +/** + */ +public class FsStats implements Iterable, Streamable, ToXContent { + + public static class Info implements Streamable { + + String path; + @Nullable + String mount; + @Nullable + String dev; + long total = -1; + long free = -1; + long available = -1; + long diskReads = -1; + long diskWrites = -1; + long diskReadBytes = -1; + long diskWriteBytes = -1; + double diskQueue = -1; + double diskServiceTime = -1; + + @Override + public void readFrom(StreamInput in) throws IOException { + path = in.readUTF(); + mount = in.readOptionalUTF(); + dev = in.readOptionalUTF(); + total = in.readLong(); + free = in.readLong(); + available = in.readLong(); + diskReads = in.readLong(); + diskWrites = in.readLong(); + diskReadBytes = in.readLong(); + diskWriteBytes = in.readLong(); + diskQueue = in.readDouble(); + diskServiceTime = in.readDouble(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeUTF(path); + out.writeOptionalUTF(mount); + out.writeOptionalUTF(dev); + out.writeLong(total); + out.writeLong(free); + out.writeLong(available); + out.writeLong(diskReads); + out.writeLong(diskWrites); + out.writeLong(diskReadBytes); + out.writeLong(diskWriteBytes); + out.writeDouble(diskQueue); + out.writeDouble(diskServiceTime); + } + + public ByteSizeValue total() { + return new ByteSizeValue(total); + } + + public ByteSizeValue getTotal() { + return total(); + } + + public ByteSizeValue free() { + return new ByteSizeValue(free); + } + + public ByteSizeValue getFree() { + return free(); + } + + public ByteSizeValue available() { + return new ByteSizeValue(available); + } + + public ByteSizeValue getAvailable() { + return available(); + } + + public long diskReads() { + return this.diskReads; + } + + public long getDiskReads() { + return this.diskReads; + } + + public long diskWrites() { + return this.diskWrites; + } + + public long getDiskWrites() { + return this.diskWrites; + } + + public long diskReadSizeInBytes() { + return diskReadBytes; + } + + public long getDiskReadSizeInBytes() { + return diskReadBytes; + } + + public ByteSizeValue diskReadSizeSize() { + return new ByteSizeValue(diskReadBytes); + } + + public ByteSizeValue getDiskReadSizeSize() { + return new ByteSizeValue(diskReadBytes); + } + + public long diskWriteSizeInBytes() { + return diskWriteBytes; + } + + public long getDiskWriteSizeInBytes() { + return diskWriteBytes; + } + + public ByteSizeValue diskWriteSizeSize() { + return new ByteSizeValue(diskWriteBytes); + } + + public ByteSizeValue getDiskWriteSizeSize() { + return new ByteSizeValue(diskWriteBytes); + } + + public double diskQueue() { + return diskQueue; + } + + public double getDiskQueue() { + return diskQueue; + } + + public double diskServiceTime() { + return diskServiceTime; + } + + public double getDiskServiceTime() { + return diskServiceTime; + } + } + + long timestamp; + Info[] infos; + + FsStats() { + + } + + FsStats(long timestamp, Info[] infos) { + this.timestamp = timestamp; + this.infos = infos; + } + + public long timestamp() { + return timestamp; + } + + public long getTimestamp() { + return timestamp(); + } + + @Override + public Iterator iterator() { + return Iterators.forArray(infos); + } + + public static FsStats readFsStats(StreamInput in) throws IOException { + FsStats stats = new FsStats(); + stats.readFrom(in); + return stats; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + timestamp = in.readVLong(); + infos = new Info[in.readVInt()]; + for (int i = 0; i < infos.length; i++) { + infos[i] = new Info(); + infos[i].readFrom(in); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(timestamp); + out.writeVInt(infos.length); + for (Info info : infos) { + info.writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("fs"); + builder.field("timestamp", timestamp); + builder.startArray("data"); + for (Info info : infos) { + builder.startObject(); + builder.field("path", info.path); + if (info.mount != null) { + builder.field("mount", info.mount); + } + if (info.dev != null) { + builder.field("dev", info.dev); + } + + if (info.total != -1) { + builder.field("total", info.total().toString()); + builder.field("total_in_bytes", info.total); + } + if (info.free != -1) { + builder.field("free", info.free().toString()); + builder.field("free_in_bytes", info.free); + } + if (info.available != -1) { + builder.field("available", info.available().toString()); + builder.field("available_in_bytes", info.available); + } + + if (info.diskReads != -1) { + builder.field("disk_reads", info.diskReads); + } + if (info.diskWrites != -1) { + builder.field("disk_writes", info.diskWrites); + } + + if (info.diskReadBytes != -1) { + builder.field("disk_read_size", info.diskReadSizeSize().toString()); + builder.field("disk_read_size_bytes", info.diskReadSizeInBytes()); + } + if (info.diskWriteBytes != -1) { + builder.field("disk_write_size", info.diskWriteSizeSize().toString()); + builder.field("disk_write_size_bytes", info.diskWriteSizeInBytes()); + } + + if (info.diskQueue != -1) { + builder.field("disk_queue", Strings.format1Decimals(info.diskQueue, "")); + } + if (info.diskServiceTime != -1) { + builder.field("disk_service_time", Strings.format1Decimals(info.diskServiceTime, "")); + } + + builder.endObject(); + } + builder.endArray(); + builder.endObject(); + return builder; + } +} diff --git a/src/main/java/org/elasticsearch/monitor/fs/JmxFsProbe.java b/src/main/java/org/elasticsearch/monitor/fs/JmxFsProbe.java new file mode 100644 index 00000000000..252d6181bfa --- /dev/null +++ b/src/main/java/org/elasticsearch/monitor/fs/JmxFsProbe.java @@ -0,0 +1,59 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.monitor.fs; + +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.NodeEnvironment; + +import java.io.File; + +/** + */ +public class JmxFsProbe extends AbstractComponent implements FsProbe { + + private final NodeEnvironment nodeEnv; + + @Inject + public JmxFsProbe(Settings settings, NodeEnvironment nodeEnv) { + super(settings); + this.nodeEnv = nodeEnv; + } + + @Override + public FsStats stats() { + if (!nodeEnv.hasNodeFile()) { + return new FsStats(System.currentTimeMillis(), new FsStats.Info[0]); + } + File[] dataLocations = nodeEnv.nodeDataLocations(); + FsStats.Info[] infos = new FsStats.Info[dataLocations.length]; + for (int i = 0; i < dataLocations.length; i++) { + File dataLocation = dataLocations[i]; + FsStats.Info info = new FsStats.Info(); + info.path = dataLocation.getAbsolutePath(); + info.total = dataLocation.getTotalSpace(); + info.free = dataLocation.getFreeSpace(); + info.available = dataLocation.getUsableSpace(); + infos[i] = info; + } + return new FsStats(System.currentTimeMillis(), infos); + } +} diff --git a/src/main/java/org/elasticsearch/monitor/fs/SigarFsProbe.java b/src/main/java/org/elasticsearch/monitor/fs/SigarFsProbe.java new file mode 100644 index 00000000000..a40cc954573 --- /dev/null +++ b/src/main/java/org/elasticsearch/monitor/fs/SigarFsProbe.java @@ -0,0 +1,94 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.monitor.fs; + +import com.google.common.collect.Maps; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.monitor.sigar.SigarService; +import org.hyperic.sigar.FileSystem; +import org.hyperic.sigar.FileSystemUsage; +import org.hyperic.sigar.SigarException; + +import java.io.File; +import java.util.Map; + +/** + */ +public class SigarFsProbe extends AbstractComponent implements FsProbe { + + private final NodeEnvironment nodeEnv; + + private final SigarService sigarService; + + private Map fileSystems = Maps.newHashMap(); + + @Inject + public SigarFsProbe(Settings settings, NodeEnvironment nodeEnv, SigarService sigarService) { + super(settings); + this.nodeEnv = nodeEnv; + this.sigarService = sigarService; + } + + @Override + public synchronized FsStats stats() { + if (!nodeEnv.hasNodeFile()) { + return new FsStats(System.currentTimeMillis(), new FsStats.Info[0]); + } + File[] dataLocations = nodeEnv.nodeDataLocations(); + FsStats.Info[] infos = new FsStats.Info[dataLocations.length]; + for (int i = 0; i < dataLocations.length; i++) { + File dataLocation = dataLocations[i]; + + FsStats.Info info = new FsStats.Info(); + info.path = dataLocation.getAbsolutePath(); + + try { + FileSystem fileSystem = fileSystems.get(dataLocation); + if (fileSystem == null) { + fileSystem = sigarService.sigar().getFileSystemMap().getMountPoint(dataLocation.getPath()); + fileSystems.put(dataLocation, fileSystem); + } + + FileSystemUsage fileSystemUsage = sigarService.sigar().getFileSystemUsage(fileSystem.getDirName()); + info.mount = fileSystem.getDirName(); + info.dev = fileSystem.getDevName(); + // total/free/available seem to be in megabytes? + info.total = fileSystemUsage.getTotal() * 1024; + info.free = fileSystemUsage.getFree() * 1024; + info.available = fileSystemUsage.getAvail() * 1024; + info.diskReads = fileSystemUsage.getDiskReads(); + info.diskWrites = fileSystemUsage.getDiskWrites(); + info.diskReadBytes = fileSystemUsage.getDiskReadBytes(); + info.diskWriteBytes = fileSystemUsage.getDiskWriteBytes(); + info.diskQueue = fileSystemUsage.getDiskQueue(); + info.diskServiceTime = fileSystemUsage.getDiskServiceTime(); + } catch (SigarException e) { + // failed... + } + + infos[i] = info; + } + + return new FsStats(System.currentTimeMillis(), infos); + } +} diff --git a/src/main/java/org/elasticsearch/node/service/NodeService.java b/src/main/java/org/elasticsearch/node/service/NodeService.java index 62d428b9878..c4b06980fcc 100644 --- a/src/main/java/org/elasticsearch/node/service/NodeService.java +++ b/src/main/java/org/elasticsearch/node/service/NodeService.java @@ -140,12 +140,13 @@ public class NodeService extends AbstractComponent { monitorService.jvmService().stats(), threadPool.stats(), monitorService.networkService().stats(), + monitorService.fsService().stats(), transportService.stats(), httpServer == null ? null : httpServer.stats() ); } - public NodeStats stats(boolean indices, boolean os, boolean process, boolean jvm, boolean threadPool, boolean network, boolean transport, boolean http) { + public NodeStats stats(boolean indices, boolean os, boolean process, boolean jvm, boolean threadPool, boolean network, boolean fs, boolean transport, boolean http) { // for indices stats we want to include previous allocated shards stats as well (it will // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats) return new NodeStats(clusterService.state().nodes().localNode(), hostname, @@ -155,6 +156,7 @@ public class NodeService extends AbstractComponent { jvm ? monitorService.jvmService().stats() : null, threadPool ? this.threadPool.stats() : null, network ? monitorService.networkService().stats() : null, + fs ? monitorService.fsService().stats() : null, transport ? transportService.stats() : null, http ? (httpServer == null ? null : httpServer.stats()) : null ); diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java index 0d78cfa8b16..df87cfb9dbc 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java @@ -64,6 +64,9 @@ public class RestNodesStatsAction extends BaseRestHandler { controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/network", new RestNetworkHandler()); controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/network", new RestNetworkHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/fs", new RestFsHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/fs", new RestFsHandler()); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/transport", new RestTransportHandler()); controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/transport", new RestTransportHandler()); @@ -85,6 +88,7 @@ public class RestNodesStatsAction extends BaseRestHandler { nodesStatsRequest.jvm(request.paramAsBoolean("jvm", nodesStatsRequest.jvm())); nodesStatsRequest.threadPool(request.paramAsBoolean("thread_pool", nodesStatsRequest.threadPool())); nodesStatsRequest.network(request.paramAsBoolean("network", nodesStatsRequest.network())); + nodesStatsRequest.fs(request.paramAsBoolean("fs", nodesStatsRequest.fs())); nodesStatsRequest.transport(request.paramAsBoolean("transport", nodesStatsRequest.transport())); nodesStatsRequest.http(request.paramAsBoolean("http", nodesStatsRequest.http())); executeNodeStats(request, channel, nodesStatsRequest); @@ -171,6 +175,15 @@ public class RestNodesStatsAction extends BaseRestHandler { } } + class RestFsHandler implements RestHandler { + @Override + public void handleRequest(final RestRequest request, final RestChannel channel) { + NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(RestActions.splitNodes(request.param("nodeId"))); + nodesStatsRequest.clear().fs(true); + executeNodeStats(request, channel, nodesStatsRequest); + } + } + class RestTransportHandler implements RestHandler { @Override public void handleRequest(final RestRequest request, final RestChannel channel) { From 5325bf4bec9e6554ca56be8d279a02c5c9e12ef5 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 18 Jan 2012 21:18:29 +0200 Subject: [PATCH 184/270] allow plugins to provide actual Module instances, not just classes to create the modules from --- .../elasticsearch/plugins/AbstractPlugin.java | 24 +++++++++++++++++++ .../plugins/IndexPluginsModule.java | 1 + .../org/elasticsearch/plugins/Plugin.java | 19 ++++++++++++++- .../elasticsearch/plugins/PluginsModule.java | 1 + .../elasticsearch/plugins/PluginsService.java | 24 +++++++++++++++++++ .../plugins/ShardsPluginsModule.java | 1 + 6 files changed, 69 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/plugins/AbstractPlugin.java b/src/main/java/org/elasticsearch/plugins/AbstractPlugin.java index 18cefc387d3..9f04fd2d4bd 100644 --- a/src/main/java/org/elasticsearch/plugins/AbstractPlugin.java +++ b/src/main/java/org/elasticsearch/plugins/AbstractPlugin.java @@ -45,6 +45,14 @@ public abstract class AbstractPlugin implements Plugin { return ImmutableList.of(); } + /** + * Defaults to return an empty list. + */ + @Override + public Collection modules(Settings settings) { + return ImmutableList.of(); + } + /** * Defaults to return an empty list. */ @@ -61,6 +69,14 @@ public abstract class AbstractPlugin implements Plugin { return ImmutableList.of(); } + /** + * Defaults to return an empty list. + */ + @Override + public Collection indexModules(Settings settings) { + return ImmutableList.of(); + } + /** * Defaults to return an empty list. */ @@ -77,6 +93,14 @@ public abstract class AbstractPlugin implements Plugin { return ImmutableList.of(); } + /** + * Defaults to return an empty list. + */ + @Override + public Collection shardModules(Settings settings) { + return ImmutableList.of(); + } + /** * Defaults to return an empty list. */ diff --git a/src/main/java/org/elasticsearch/plugins/IndexPluginsModule.java b/src/main/java/org/elasticsearch/plugins/IndexPluginsModule.java index eefb534f14b..a7dc535955e 100644 --- a/src/main/java/org/elasticsearch/plugins/IndexPluginsModule.java +++ b/src/main/java/org/elasticsearch/plugins/IndexPluginsModule.java @@ -52,6 +52,7 @@ public class IndexPluginsModule extends AbstractModule implements SpawnModules, for (Class moduleClass : modulesClasses) { modules.add(createModule(moduleClass, settings)); } + modules.addAll(pluginsService.indexModules(settings)); return modules; } diff --git a/src/main/java/org/elasticsearch/plugins/Plugin.java b/src/main/java/org/elasticsearch/plugins/Plugin.java index 6fae62a4d8c..d85d1e2ee96 100644 --- a/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -46,10 +46,17 @@ public interface Plugin { String description(); /** - * Node level modules. + * Node level modules (classes, will automatically be created). */ Collection> modules(); + /** + * Node level modules (instances) + * + * @param settings The node level settings. + */ + Collection modules(Settings settings); + /** * Node level services that will be automatically started/stopped/closed. */ @@ -60,6 +67,11 @@ public interface Plugin { */ Collection> indexModules(); + /** + * Per index modules. + */ + Collection indexModules(Settings settings); + /** * Per index services that will be automatically closed. */ @@ -70,6 +82,11 @@ public interface Plugin { */ Collection> shardModules(); + /** + * Per index shard module. + */ + Collection shardModules(Settings settings); + /** * Per index shard service that will be automatically closed. */ diff --git a/src/main/java/org/elasticsearch/plugins/PluginsModule.java b/src/main/java/org/elasticsearch/plugins/PluginsModule.java index c43aa4c66cd..53c7bcf26dc 100644 --- a/src/main/java/org/elasticsearch/plugins/PluginsModule.java +++ b/src/main/java/org/elasticsearch/plugins/PluginsModule.java @@ -52,6 +52,7 @@ public class PluginsModule extends AbstractModule implements SpawnModules, PrePr for (Class moduleClass : modulesClasses) { modules.add(createModule(moduleClass, settings)); } + modules.addAll(pluginsService.modules(settings)); return modules; } diff --git a/src/main/java/org/elasticsearch/plugins/PluginsService.java b/src/main/java/org/elasticsearch/plugins/PluginsService.java index 4f94f86983a..0df969bb3d7 100644 --- a/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -151,6 +151,14 @@ public class PluginsService extends AbstractComponent { return modules; } + public Collection modules(Settings settings) { + List modules = Lists.newArrayList(); + for (Plugin plugin : plugins.values()) { + modules.addAll(plugin.modules(settings)); + } + return modules; + } + public Collection> services() { List> services = Lists.newArrayList(); for (Plugin plugin : plugins.values()) { @@ -167,6 +175,14 @@ public class PluginsService extends AbstractComponent { return modules; } + public Collection indexModules(Settings settings) { + List modules = Lists.newArrayList(); + for (Plugin plugin : plugins.values()) { + modules.addAll(plugin.indexModules(settings)); + } + return modules; + } + public Collection> indexServices() { List> services = Lists.newArrayList(); for (Plugin plugin : plugins.values()) { @@ -183,6 +199,14 @@ public class PluginsService extends AbstractComponent { return modules; } + public Collection shardModules(Settings settings) { + List modules = Lists.newArrayList(); + for (Plugin plugin : plugins.values()) { + modules.addAll(plugin.shardModules(settings)); + } + return modules; + } + public Collection> shardServices() { List> services = Lists.newArrayList(); for (Plugin plugin : plugins.values()) { diff --git a/src/main/java/org/elasticsearch/plugins/ShardsPluginsModule.java b/src/main/java/org/elasticsearch/plugins/ShardsPluginsModule.java index 0c56b0c262d..3ee510d9eaf 100644 --- a/src/main/java/org/elasticsearch/plugins/ShardsPluginsModule.java +++ b/src/main/java/org/elasticsearch/plugins/ShardsPluginsModule.java @@ -52,6 +52,7 @@ public class ShardsPluginsModule extends AbstractModule implements SpawnModules, for (Class moduleClass : modulesClasses) { modules.add(createModule(moduleClass, settings)); } + modules.addAll(pluginsService.shardModules(settings)); return modules; } From 5beb656a16a1fe5f250ae4155b31ad809e1e0ef3 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 18 Jan 2012 21:34:33 +0200 Subject: [PATCH 185/270] have terms filter builder support iterables as well --- .../index/query/FilterBuilders.java | 12 +++++- .../index/query/TermsFilterBuilder.java | 39 ++++++++----------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/query/FilterBuilders.java b/src/main/java/org/elasticsearch/index/query/FilterBuilders.java index 9c9a759196f..724d9c6d8ba 100644 --- a/src/main/java/org/elasticsearch/index/query/FilterBuilders.java +++ b/src/main/java/org/elasticsearch/index/query/FilterBuilders.java @@ -23,8 +23,6 @@ import org.elasticsearch.common.Nullable; /** * A static factory for simple "import static" usage. - * - * */ public abstract class FilterBuilders { @@ -186,6 +184,16 @@ public abstract class FilterBuilders { return new TermsFilterBuilder(name, values); } + /** + * A filer for a field based on several terms matching on any of them. + * + * @param name The field name + * @param values The terms + */ + public static TermsFilterBuilder termsFilter(String name, Iterable values) { + return new TermsFilterBuilder(name, values); + } + /** * A filer for a field based on several terms matching on any of them. * diff --git a/src/main/java/org/elasticsearch/index/query/TermsFilterBuilder.java b/src/main/java/org/elasticsearch/index/query/TermsFilterBuilder.java index 6ec30bfc89b..685dc0a21e7 100644 --- a/src/main/java/org/elasticsearch/index/query/TermsFilterBuilder.java +++ b/src/main/java/org/elasticsearch/index/query/TermsFilterBuilder.java @@ -30,7 +30,7 @@ public class TermsFilterBuilder extends BaseFilterBuilder { private final String name; - private final Object[] values; + private final Object values; private Boolean cache; private String cacheKey; @@ -57,10 +57,7 @@ public class TermsFilterBuilder extends BaseFilterBuilder { */ public TermsFilterBuilder(String name, int... values) { this.name = name; - this.values = new Integer[values.length]; - for (int i = 0; i < values.length; i++) { - this.values[i] = values[i]; - } + this.values = values; } /** @@ -71,10 +68,7 @@ public class TermsFilterBuilder extends BaseFilterBuilder { */ public TermsFilterBuilder(String name, long... values) { this.name = name; - this.values = new Long[values.length]; - for (int i = 0; i < values.length; i++) { - this.values[i] = values[i]; - } + this.values = values; } /** @@ -85,10 +79,7 @@ public class TermsFilterBuilder extends BaseFilterBuilder { */ public TermsFilterBuilder(String name, float... values) { this.name = name; - this.values = new Float[values.length]; - for (int i = 0; i < values.length; i++) { - this.values[i] = values[i]; - } + this.values = values; } /** @@ -99,10 +90,7 @@ public class TermsFilterBuilder extends BaseFilterBuilder { */ public TermsFilterBuilder(String name, double... values) { this.name = name; - this.values = new Double[values.length]; - for (int i = 0; i < values.length; i++) { - this.values[i] = values[i]; - } + this.values = values; } /** @@ -116,6 +104,17 @@ public class TermsFilterBuilder extends BaseFilterBuilder { this.values = values; } + /** + * A filer for a field based on several terms matching on any of them. + * + * @param name The field name + * @param values The terms + */ + public TermsFilterBuilder(String name, Iterable values) { + this.name = name; + this.values = values; + } + /** * Sets the execution mode for the terms filter. Cane be either "plain", "bool" * "and". Defaults to "plain". @@ -149,11 +148,7 @@ public class TermsFilterBuilder extends BaseFilterBuilder { @Override public void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(TermsFilterParser.NAME); - builder.startArray(name); - for (Object value : values) { - builder.value(value); - } - builder.endArray(); + builder.field(name, values); if (execution != null) { builder.field("execution", execution); From d95783b158329f2ab3e0f09b111f52c5cfe0a6a8 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 18 Jan 2012 23:02:13 +0200 Subject: [PATCH 186/270] index.recovery.initial_shards is not being taken into account, closes #1623. --- .../org/elasticsearch/gateway/local/LocalGatewayAllocator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java b/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java index ac9b4171e09..7026f05f082 100644 --- a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java +++ b/src/main/java/org/elasticsearch/gateway/local/LocalGatewayAllocator.java @@ -156,7 +156,7 @@ public class LocalGatewayAllocator extends AbstractComponent implements GatewayA int requiredAllocation = 1; try { IndexMetaData indexMetaData = routingNodes.metaData().index(shard.index()); - String initialShards = indexMetaData.settings().get("recovery.initial_shards", this.initialShards); + String initialShards = indexMetaData.settings().get("index.recovery.initial_shards", this.initialShards); if ("quorum".equals(initialShards)) { if (indexMetaData.numberOfReplicas() > 1) { requiredAllocation = ((1 + indexMetaData.numberOfReplicas()) / 2) + 1; From 2eeb609353070fd518b5a9a0774c63c8892bc110 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 19 Jan 2012 00:22:27 +0200 Subject: [PATCH 187/270] Thread Pool: Add a dedicated thread pool for refresh, closes #1624. --- .../action/admin/indices/refresh/TransportRefreshAction.java | 2 +- .../elasticsearch/index/shard/service/InternalIndexShard.java | 2 +- src/main/java/org/elasticsearch/threadpool/ThreadPool.java | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java b/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java index 7f64aa4d6f4..c894e4f94c5 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java @@ -63,7 +63,7 @@ public class TransportRefreshAction extends TransportBroadcastOperationAction Date: Wed, 18 Jan 2012 23:53:02 +0100 Subject: [PATCH 188/270] updatable default TTL in mapping --- .../index/mapper/internal/TTLFieldMapper.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java index d9f672e4c72..06908dfe1cf 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java @@ -218,6 +218,11 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R @Override public void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappingException { - // do nothing here, no merging, but also no exception + TTLFieldMapper ttlMergeWith = (TTLFieldMapper) mergeWith; + if (!mergeContext.mergeFlags().simulate()) { + if (ttlMergeWith.defaultTTL != -1) { + this.defaultTTL = ttlMergeWith.defaultTTL; + } + } } } From a1a30226caaef4667cb7d85a4702ecf6a6bb0703 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 19 Jan 2012 15:42:23 +0200 Subject: [PATCH 189/270] Removing a node with TRACE logging enabled causes cluster state not to be properly updated, closes #1626. --- .../elasticsearch/cluster/routing/RoutingNode.java | 13 ++++++++----- .../elasticsearch/cluster/routing/RoutingNodes.java | 2 +- .../routing/allocation/AllocationService.java | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java b/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java index 4728169fb71..872f6b6a55b 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java +++ b/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java @@ -31,15 +31,18 @@ import static com.google.common.collect.Lists.newArrayList; */ public class RoutingNode implements Iterable { + private final String nodeId; + private final DiscoveryNode node; private final List shards; - public RoutingNode(DiscoveryNode node) { - this(node, new ArrayList()); + public RoutingNode(String nodeId, DiscoveryNode node) { + this(nodeId, node, new ArrayList()); } - public RoutingNode(DiscoveryNode node, List shards) { + public RoutingNode(String nodeId, DiscoveryNode node, List shards) { + this.nodeId = nodeId; this.node = node; this.shards = shards; } @@ -54,7 +57,7 @@ public class RoutingNode implements Iterable { } public String nodeId() { - return this.node.id(); + return this.nodeId; } public List shards() { @@ -140,7 +143,7 @@ public class RoutingNode implements Iterable { public String prettyPrint() { StringBuilder sb = new StringBuilder(); - sb.append("-----node_id[").append(node.id()).append("]\n"); + sb.append("-----node_id[").append(nodeId).append("][" + (node == null ? "X" : "V") + "]\n"); for (MutableShardRouting entry : shards) { sb.append("--------").append(entry.shortSummary()).append('\n'); } diff --git a/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java index cf147fcc5e5..e01fb698da9 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java +++ b/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java @@ -86,7 +86,7 @@ public class RoutingNodes implements Iterable { } for (Map.Entry> entry : nodesToShards.entrySet()) { String nodeId = entry.getKey(); - this.nodesToShards.put(nodeId, new RoutingNode(clusterState.nodes().get(nodeId), entry.getValue())); + this.nodesToShards.put(nodeId, new RoutingNode(nodeId, clusterState.nodes().get(nodeId), entry.getValue())); } } diff --git a/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index c9301ff90f9..3af0cc04e1a 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -246,7 +246,7 @@ public class AllocationService extends AbstractComponent { private void applyNewNodes(RoutingNodes routingNodes, Iterable liveNodes) { for (DiscoveryNode node : liveNodes) { if (!routingNodes.nodesToShards().containsKey(node.id())) { - RoutingNode routingNode = new RoutingNode(node); + RoutingNode routingNode = new RoutingNode(node.id(), node); routingNodes.nodesToShards().put(node.id(), routingNode); } } From 62809bb62a33f517f02565401b6c1b803dd99bb6 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 19 Jan 2012 23:32:59 +0200 Subject: [PATCH 190/270] Upgrade to netty 3.3.0 --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 817f98c9b46..7ea73599120 100644 --- a/pom.xml +++ b/pom.xml @@ -151,9 +151,9 @@ - org.jboss.netty + io.netty netty - 3.2.7.Final + 3.3.0.Final compile @@ -310,7 +310,7 @@ org.codehaus.jackson:jackson-smile org.yaml:snakeyaml joda-time:joda-time - org.jboss.netty:netty + io.netty:netty From 942b427940f8dbc3695e391e2912969ded5625d8 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 22 Jan 2012 23:34:34 +0200 Subject: [PATCH 191/270] Local Gateway: Store specific index metadata under dedicated index locations, closes #1631. --- config/elasticsearch.yml | 8 +- .../state/TransportClusterStateAction.java | 2 +- .../cluster/metadata/IndexMetaData.java | 34 +- .../metadata/IndexTemplateMetaData.java | 26 + .../cluster/metadata/MetaData.java | 51 +- .../metadata/MetaDataCreateIndexService.java | 2 +- .../metadata/MetaDataMappingService.java | 6 +- .../cluster/routing/RoutingService.java | 16 +- .../service/InternalClusterService.java | 211 ++++---- .../common/collect/MapBuilder.java | 5 + .../discovery/zen/ZenDiscovery.java | 26 +- .../elasticsearch/env/NodeEnvironment.java | 23 +- .../org/elasticsearch/gateway/Gateway.java | 2 +- .../elasticsearch/gateway/GatewayService.java | 30 +- .../gateway/local/LocalGateway.java | 324 ++++--------- .../gateway/local/LocalGatewayMetaState.java | 131 ----- .../gateway/local/LocalGatewayModule.java | 3 + .../state/meta/LocalGatewayMetaState.java | 454 ++++++++++++++++++ .../TransportNodesListGatewayMetaState.java | 27 +- .../state/shards/LocalGatewayShardsState.java | 63 +-- .../gateway/shared/SharedStorageGateway.java | 15 +- .../test/integration/AbstractNodesTests.java | 4 + .../cluster/MinimumMasterNodesTests.java | 2 +- 23 files changed, 864 insertions(+), 601 deletions(-) delete mode 100644 src/main/java/org/elasticsearch/gateway/local/LocalGatewayMetaState.java create mode 100644 src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java rename src/main/java/org/elasticsearch/gateway/local/{ => state/meta}/TransportNodesListGatewayMetaState.java (91%) diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index 9ff4d24e338..4789ccfd1e7 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -234,7 +234,8 @@ # gateway.type: local # Settings below control how and when to start the initial recovery process on -# a full cluster restart (to reuse as much local data as possible). +# a full cluster restart (to reuse as much local data as possible when using shared +# gateway). # Allow recovery process after N nodes in a cluster are up: # @@ -246,7 +247,8 @@ # gateway.recover_after_time: 5m # Set how many nodes are expected in this cluster. Once these N nodes -# are up, begin recovery process immediately: +# are up (and recover_after_nodes is met), begin recovery process immediately +# (without waiting for recover_after_time to expire): # # gateway.expected_nodes: 2 @@ -284,7 +286,7 @@ # Set to ensure a node sees N other master eligible nodes to be considered # operational within the cluster. Set this option to a higher value (2-4) -# for large clusters: +# for large clusters (>3 nodes): # # discovery.zen.minimum_master_nodes: 1 diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java index 48f3f4d0de0..a31b564186a 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -99,7 +99,7 @@ public class TransportClusterStateAction extends TransportMasterNodeOperationAct for (String filteredIndex : indices) { IndexMetaData indexMetaData = currentState.metaData().index(filteredIndex); if (indexMetaData != null) { - mdBuilder.put(indexMetaData); + mdBuilder.put(indexMetaData, false); } } } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 46a77fe43ec..3ed61adf271 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -117,6 +117,7 @@ public class IndexMetaData { public static final String SETTING_READ_ONLY = "index.blocks.read_only"; private final String index; + private final long version; private final State state; @@ -131,10 +132,11 @@ public class IndexMetaData { private final DiscoveryNodeFilters includeFilters; private final DiscoveryNodeFilters excludeFilters; - private IndexMetaData(String index, State state, Settings settings, ImmutableMap mappings, ImmutableMap aliases) { + private IndexMetaData(String index, long version, State state, Settings settings, ImmutableMap mappings, ImmutableMap aliases) { Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, -1) != -1, "must specify numberOfShards for index [" + index + "]"); Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1) != -1, "must specify numberOfReplicas for index [" + index + "]"); this.index = index; + this.version = version; this.state = state; this.settings = settings; this.mappings = mappings; @@ -164,6 +166,14 @@ public class IndexMetaData { return index(); } + public long version() { + return this.version; + } + + public long getVersion() { + return this.version; + } + public State state() { return this.state; } @@ -274,6 +284,8 @@ public class IndexMetaData { private State state = State.OPEN; + private long version = 1; + private Settings settings = ImmutableSettings.Builder.EMPTY_SETTINGS; private MapBuilder mappings = MapBuilder.newMapBuilder(); @@ -290,6 +302,7 @@ public class IndexMetaData { mappings.putAll(indexMetaData.mappings); aliases.putAll(indexMetaData.aliases); this.state = indexMetaData.state; + this.version = indexMetaData.version; } public String index() { @@ -364,6 +377,15 @@ public class IndexMetaData { return this; } + public long version() { + return this.version; + } + + public Builder version(long version) { + this.version = version; + return this; + } + public IndexMetaData build() { MapBuilder tmpAliases = aliases; Settings tmpSettings = settings; @@ -381,12 +403,13 @@ public class IndexMetaData { tmpSettings = ImmutableSettings.settingsBuilder().put(settings).putArray("index.aliases").build(); } - return new IndexMetaData(index, state, tmpSettings, mappings.immutableMap(), tmpAliases.immutableMap()); + return new IndexMetaData(index, version, state, tmpSettings, mappings.immutableMap(), tmpAliases.immutableMap()); } public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(indexMetaData.index(), XContentBuilder.FieldCaseConversion.NONE); + builder.field("version", indexMetaData.version()); builder.field("state", indexMetaData.state().toString().toLowerCase()); builder.startObject("settings"); @@ -416,6 +439,9 @@ public class IndexMetaData { } public static IndexMetaData fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } Builder builder = new Builder(parser.currentName()); String currentFieldName = null; @@ -449,6 +475,8 @@ public class IndexMetaData { } else if (token.isValue()) { if ("state".equals(currentFieldName)) { builder.state(State.fromString(parser.text())); + } else if ("version".equals(currentFieldName)) { + builder.version(parser.longValue()); } } } @@ -457,6 +485,7 @@ public class IndexMetaData { public static IndexMetaData readFrom(StreamInput in) throws IOException { Builder builder = new Builder(in.readUTF()); + builder.version(in.readLong()); builder.state(State.fromId(in.readByte())); builder.settings(readSettingsFromStream(in)); int mappingsSize = in.readVInt(); @@ -474,6 +503,7 @@ public class IndexMetaData { public static void writeTo(IndexMetaData indexMetaData, StreamOutput out) throws IOException { out.writeUTF(indexMetaData.index()); + out.writeLong(indexMetaData.version()); out.writeByte(indexMetaData.state().id()); writeSettingsToStream(indexMetaData.settings(), out); out.writeVInt(indexMetaData.mappings().size()); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index f79e8ad623a..3026d4e753b 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -101,6 +101,32 @@ public class IndexTemplateMetaData { return new Builder(name); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + IndexTemplateMetaData that = (IndexTemplateMetaData) o; + + if (order != that.order) return false; + if (!mappings.equals(that.mappings)) return false; + if (!name.equals(that.name)) return false; + if (!settings.equals(that.settings)) return false; + if (!template.equals(that.template)) return false; + + return true; + } + + @Override + public int hashCode() { + int result = name.hashCode(); + result = 31 * result + order; + result = 31 * result + template.hashCode(); + result = 31 * result + settings.hashCode(); + result = 31 * result + mappings.hashCode(); + return result; + } + public static class Builder { private String name; diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 16ff991aacd..88a00d8e833 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -628,6 +628,12 @@ public class MetaData implements Iterable { return indices.values().iterator(); } + public static boolean isGlobalStateEquals(MetaData metaData1, MetaData metaData2) { + if (!metaData1.persistentSettings.equals(metaData2.persistentSettings)) return false; + if (!metaData1.templates.equals(metaData2.templates())) return false; + return true; + } + public static Builder builder() { return new Builder(); } @@ -658,10 +664,21 @@ public class MetaData implements Iterable { } public Builder put(IndexMetaData.Builder indexMetaDataBuilder) { - return put(indexMetaDataBuilder.build()); + // we know its a new one, increment the version and store + indexMetaDataBuilder.version(indexMetaDataBuilder.version() + 1); + IndexMetaData indexMetaData = indexMetaDataBuilder.build(); + indices.put(indexMetaData.index(), indexMetaData); + return this; } - public Builder put(IndexMetaData indexMetaData) { + public Builder put(IndexMetaData indexMetaData, boolean incrementVersion) { + if (indices.get(indexMetaData.index()) == indexMetaData) { + return this; + } + // if we put a new index metadata, increment its version + if (incrementVersion) { + indexMetaData = IndexMetaData.newIndexMetaDataBuilder(indexMetaData).version(indexMetaData.version() + 1).build(); + } indices.put(indexMetaData.index(), indexMetaData); return this; } @@ -675,6 +692,11 @@ public class MetaData implements Iterable { return this; } + public Builder removeAllIndices() { + indices.clear(); + return this; + } + public Builder put(IndexTemplateMetaData.Builder template) { return put(template.build()); } @@ -699,8 +721,7 @@ public class MetaData implements Iterable { throw new IndexMissingException(new Index(index)); } put(IndexMetaData.newIndexMetaDataBuilder(indexMetaData) - .settings(settingsBuilder().put(indexMetaData.settings()).put(settings)) - .build()); + .settings(settingsBuilder().put(indexMetaData.settings()).put(settings))); } return this; } @@ -714,7 +735,7 @@ public class MetaData implements Iterable { if (indexMetaData == null) { throw new IndexMissingException(new Index(index)); } - put(IndexMetaData.newIndexMetaDataBuilder(indexMetaData).numberOfReplicas(numberOfReplicas).build()); + put(IndexMetaData.newIndexMetaDataBuilder(indexMetaData).numberOfReplicas(numberOfReplicas)); } return this; } @@ -757,6 +778,8 @@ public class MetaData implements Iterable { public static void toXContent(MetaData metaData, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject("meta-data"); + builder.field("version", metaData.version()); + if (!metaData.persistentSettings().getAsMap().isEmpty()) { builder.startObject("settings"); for (Map.Entry entry : metaData.persistentSettings().getAsMap().entrySet()) { @@ -771,11 +794,13 @@ public class MetaData implements Iterable { } builder.endObject(); - builder.startObject("indices"); - for (IndexMetaData indexMetaData : metaData) { - IndexMetaData.Builder.toXContent(indexMetaData, builder, params); + if (!metaData.indices().isEmpty()) { + builder.startObject("indices"); + for (IndexMetaData indexMetaData : metaData) { + IndexMetaData.Builder.toXContent(indexMetaData, builder, params); + } + builder.endObject(); } - builder.endObject(); builder.endObject(); } @@ -809,13 +834,17 @@ public class MetaData implements Iterable { builder.persistentSettings(settingsBuilder.build()); } else if ("indices".equals(currentFieldName)) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - builder.put(IndexMetaData.Builder.fromXContent(parser)); + builder.put(IndexMetaData.Builder.fromXContent(parser), false); } } else if ("templates".equals(currentFieldName)) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { builder.put(IndexTemplateMetaData.Builder.fromXContent(parser)); } } + } else if (token.isValue()) { + if ("version".equals(currentFieldName)) { + builder.version = parser.longValue(); + } } } return builder.build(); @@ -828,7 +857,7 @@ public class MetaData implements Iterable { builder.persistentSettings(readSettingsFromStream(in)); int size = in.readVInt(); for (int i = 0; i < size; i++) { - builder.put(IndexMetaData.Builder.readFrom(in)); + builder.put(IndexMetaData.Builder.readFrom(in), false); } size = in.readVInt(); for (int i = 0; i < size; i++) { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 7fd7c6fab47..5138cdaf0c2 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -256,7 +256,7 @@ public class MetaDataCreateIndexService extends AbstractComponent { MetaData newMetaData = newMetaDataBuilder() .metaData(currentState.metaData()) - .put(indexMetaData) + .put(indexMetaData, false) .build(); logger.info("[{}] creating index, cause [{}], shards [{}]/[{}], mappings {}", request.index, request.cause, indexMetaData.numberOfShards(), indexMetaData.numberOfReplicas(), mappings.keySet()); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java index aff6d5fec4a..2561cb0cf68 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java @@ -264,6 +264,7 @@ public class MetaDataMappingService extends AbstractComponent { } public void putMapping(final PutRequest request, final Listener listener) { + final AtomicBoolean notifyOnPostProcess = new AtomicBoolean(); clusterService.submitStateUpdateTask("put-mapping [" + request.mappingType + "]", new ProcessedClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { @@ -394,7 +395,7 @@ public class MetaDataMappingService extends AbstractComponent { } if (counter == 0) { - listener.onResponse(new Response(true)); + notifyOnPostProcess.set(true); return updatedState; } mappingCreatedAction.add(new CountDownListener(counter, listener), request.timeout); @@ -411,6 +412,9 @@ public class MetaDataMappingService extends AbstractComponent { @Override public void clusterStateProcessed(ClusterState clusterState) { + if (notifyOnPostProcess.get()) { + listener.onResponse(new Response(true)); + } } }); } diff --git a/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java b/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java index 337b54d415f..639444bd34f 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java +++ b/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java @@ -61,15 +61,19 @@ public class RoutingService extends AbstractLifecycleComponent i this.clusterService = clusterService; this.allocationService = allocationService; this.schedule = componentSettings.getAsTime("schedule", timeValueSeconds(10)); - } - - @Override - protected void doStart() throws ElasticSearchException { clusterService.addPriority(this); } + @Override + protected void doStart() throws ElasticSearchException { + } + @Override protected void doStop() throws ElasticSearchException { + } + + @Override + protected void doClose() throws ElasticSearchException { if (scheduledRoutingTableFuture != null) { scheduledRoutingTableFuture.cancel(true); scheduledRoutingTableFuture = null; @@ -77,10 +81,6 @@ public class RoutingService extends AbstractLifecycleComponent i clusterService.remove(this); } - @Override - protected void doClose() throws ElasticSearchException { - } - @Override public void clusterChanged(ClusterChangedEvent event) { if (event.source().equals(CLUSTER_UPDATE_TASK_SOURCE)) { diff --git a/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java b/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java index cfc53b4afd1..a59cf29a220 100644 --- a/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java +++ b/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java @@ -164,6 +164,8 @@ public class InternalClusterService extends AbstractLifecycleComponent it = onGoingTimeouts.iterator(); it.hasNext(); ) { NotifyTimeout timeout = it.next(); if (timeout.listener.equals(listener)) { @@ -204,109 +206,126 @@ public class InternalClusterService extends AbstractLifecycleComponent 0) { - logger.info("{}, reason: {}", summary, source); - } - } - - // TODO, do this in parallel (and wait) - for (DiscoveryNode node : nodesDelta.addedNodes()) { - if (!nodeRequiresConnection(node)) { - continue; - } - try { - transportService.connectToNode(node); - } catch (Exception e) { - // the fault detection will detect it as failed as well - logger.warn("failed to connect to node [" + node + "]", e); - } - } - - for (ClusterStateListener listener : priorityClusterStateListeners) { - listener.clusterChanged(clusterChangedEvent); - } - for (ClusterStateListener listener : clusterStateListeners) { - listener.clusterChanged(clusterChangedEvent); - } - for (ClusterStateListener listener : lastClusterStateListeners) { - listener.clusterChanged(clusterChangedEvent); - } - - if (!nodesDelta.removedNodes().isEmpty()) { - threadPool.cached().execute(new Runnable() { - @Override - public void run() { - for (DiscoveryNode node : nodesDelta.removedNodes()) { - transportService.disconnectFromNode(node); - } - } - }); - } - - // if we are the master, publish the new state to all nodes - if (clusterState.nodes().localNodeMaster()) { - discoveryService.publish(clusterState); - } - - if (updateTask instanceof ProcessedClusterStateUpdateTask) { - ((ProcessedClusterStateUpdateTask) updateTask).clusterStateProcessed(clusterState); - } - - logger.debug("processing [{}]: done applying updated cluster_state", source); - } catch (Exception e) { - StringBuilder sb = new StringBuilder("failed to apply updated cluster state:\nversion [").append(clusterState.version()).append("], source [").append(source).append("]\n"); - sb.append(clusterState.nodes().prettyPrint()); - sb.append(clusterState.routingTable().prettyPrint()); - sb.append(clusterState.readOnlyRoutingNodes().prettyPrint()); - logger.warn(sb.toString(), e); - } - } else { + if (previousClusterState == newClusterState) { logger.debug("processing [{}]: no change in cluster_state", source); + return; + } + + try { + if (newClusterState.nodes().localNodeMaster()) { + // only the master controls the version numbers + Builder builder = ClusterState.builder().state(newClusterState).version(newClusterState.version() + 1); + if (previousClusterState.routingTable() != newClusterState.routingTable()) { + builder.routingTable(RoutingTable.builder().routingTable(newClusterState.routingTable()).version(newClusterState.routingTable().version() + 1)); + } + if (previousClusterState.metaData() != newClusterState.metaData()) { + builder.metaData(MetaData.builder().metaData(newClusterState.metaData()).version(newClusterState.metaData().version() + 1)); + } + newClusterState = builder.build(); + } else { + if (previousClusterState.blocks().hasGlobalBlock(Discovery.NO_MASTER_BLOCK) && !newClusterState.blocks().hasGlobalBlock(Discovery.NO_MASTER_BLOCK)) { + // force an update, its a fresh update from the master as we transition from a start of not having a master to having one + // have a fresh instances of routing and metadata to remove the chance that version might be the same + Builder builder = ClusterState.builder().state(newClusterState); + builder.routingTable(RoutingTable.builder().routingTable(newClusterState.routingTable())); + builder.metaData(MetaData.builder().metaData(newClusterState.metaData())); + newClusterState = builder.build(); + logger.debug("got first state from fresh master [{}]", newClusterState.nodes().masterNodeId()); + } else if (newClusterState.version() < previousClusterState.version()) { + // we got this cluster state from the master, filter out based on versions (don't call listeners) + logger.debug("got old cluster state [" + newClusterState.version() + "<" + previousClusterState.version() + "] from source [" + source + "], ignoring"); + return; + } + } + + if (logger.isTraceEnabled()) { + StringBuilder sb = new StringBuilder("cluster state updated:\nversion [").append(newClusterState.version()).append("], source [").append(source).append("]\n"); + sb.append(newClusterState.nodes().prettyPrint()); + sb.append(newClusterState.routingTable().prettyPrint()); + sb.append(newClusterState.readOnlyRoutingNodes().prettyPrint()); + logger.trace(sb.toString()); + } else if (logger.isDebugEnabled()) { + logger.debug("cluster state updated, version [{}], source [{}]", newClusterState.version(), source); + } + + ClusterChangedEvent clusterChangedEvent = new ClusterChangedEvent(source, newClusterState, previousClusterState); + // new cluster state, notify all listeners + final DiscoveryNodes.Delta nodesDelta = clusterChangedEvent.nodesDelta(); + if (nodesDelta.hasChanges() && logger.isInfoEnabled()) { + String summary = nodesDelta.shortSummary(); + if (summary.length() > 0) { + logger.info("{}, reason: {}", summary, source); + } + } + + // TODO, do this in parallel (and wait) + for (DiscoveryNode node : nodesDelta.addedNodes()) { + if (!nodeRequiresConnection(node)) { + continue; + } + try { + transportService.connectToNode(node); + } catch (Exception e) { + // the fault detection will detect it as failed as well + logger.warn("failed to connect to node [" + node + "]", e); + } + } + + // if we are the master, publish the new state to all nodes + // we publish here before we send a notification to all the listeners, since if it fails + // we don't want to notify + if (newClusterState.nodes().localNodeMaster()) { + discoveryService.publish(newClusterState); + } + + // update the current cluster state + clusterState = newClusterState; + + for (ClusterStateListener listener : priorityClusterStateListeners) { + listener.clusterChanged(clusterChangedEvent); + } + for (ClusterStateListener listener : clusterStateListeners) { + listener.clusterChanged(clusterChangedEvent); + } + for (ClusterStateListener listener : lastClusterStateListeners) { + listener.clusterChanged(clusterChangedEvent); + } + + if (!nodesDelta.removedNodes().isEmpty()) { + threadPool.cached().execute(new Runnable() { + @Override + public void run() { + for (DiscoveryNode node : nodesDelta.removedNodes()) { + transportService.disconnectFromNode(node); + } + } + }); + } + + + if (updateTask instanceof ProcessedClusterStateUpdateTask) { + ((ProcessedClusterStateUpdateTask) updateTask).clusterStateProcessed(newClusterState); + } + + logger.debug("processing [{}]: done applying updated cluster_state", source); + } catch (Exception e) { + StringBuilder sb = new StringBuilder("failed to apply updated cluster state:\nversion [").append(newClusterState.version()).append("], source [").append(source).append("]\n"); + sb.append(newClusterState.nodes().prettyPrint()); + sb.append(newClusterState.routingTable().prettyPrint()); + sb.append(newClusterState.readOnlyRoutingNodes().prettyPrint()); + logger.warn(sb.toString(), e); } } }); diff --git a/src/main/java/org/elasticsearch/common/collect/MapBuilder.java b/src/main/java/org/elasticsearch/common/collect/MapBuilder.java index 82176aa5362..e69e991f17c 100644 --- a/src/main/java/org/elasticsearch/common/collect/MapBuilder.java +++ b/src/main/java/org/elasticsearch/common/collect/MapBuilder.java @@ -59,6 +59,11 @@ public class MapBuilder { return this; } + public MapBuilder clear() { + this.map.clear(); + return this; + } + public V get(K key) { return map.get(key); } diff --git a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java index efaed31d0cb..d1f1c72d47f 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java @@ -315,9 +315,9 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen continue; } // send join request - ClusterState clusterState; + ClusterState joinClusterStateX; try { - clusterState = membership.sendJoinRequestBlocking(masterNode, localNode, pingTimeout); + joinClusterStateX = membership.sendJoinRequestBlocking(masterNode, localNode, pingTimeout); } catch (Exception e) { if (e instanceof ElasticSearchException) { logger.info("failed to send join request to master [{}], reason [{}]", masterNode, ((ElasticSearchException) e).getDetailedMessage()); @@ -332,26 +332,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen continue; } masterFD.start(masterNode, "initial_join"); - - // we update the metadata once we managed to join, so we pre-create indices and so on (no shards allocation) - final MetaData metaData = clusterState.metaData(); - // sync also the version with the version the master currently has, so the next update will be applied - final long version = clusterState.version(); - clusterService.submitStateUpdateTask("zen-disco-join (detected master)", new ProcessedClusterStateUpdateTask() { - @Override - public ClusterState execute(ClusterState currentState) { - ClusterBlocks clusterBlocks = ClusterBlocks.builder().blocks(currentState.blocks()).removeGlobalBlock(NO_MASTER_BLOCK).build(); - // make sure we have the local node id set, we might need it as a result of the new metadata - DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.newNodesBuilder().putAll(currentState.nodes()).put(localNode).localNodeId(localNode.id()); - return newClusterStateBuilder().state(currentState).nodes(nodesBuilder).blocks(clusterBlocks).metaData(metaData).version(version).build(); - } - - @Override - public void clusterStateProcessed(ClusterState clusterState) { - // don't send initial state event, since we want to get the cluster state from the master that includes us first -// sendInitialStateEventIfNeeded(); - } - }); + // no need to submit the received cluster state, we will get it from the master when it publishes + // the fact that we joined } } } diff --git a/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/src/main/java/org/elasticsearch/env/NodeEnvironment.java index fa71a28a665..2ca6fdbe26b 100644 --- a/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -20,6 +20,7 @@ package org.elasticsearch.env; import com.google.common.collect.Sets; +import com.google.common.primitives.Ints; import org.apache.lucene.store.Lock; import org.apache.lucene.store.NativeFSLockFactory; import org.elasticsearch.ElasticSearchIllegalStateException; @@ -177,6 +178,23 @@ public class NodeEnvironment extends AbstractComponent { return shardLocations; } + public Set finalAllIndices() throws Exception { + if (nodeFiles == null || locks == null) { + throw new ElasticSearchIllegalStateException("node is not configured to store local location"); + } + Set indices = Sets.newHashSet(); + for (File indicesLocation : nodeIndicesLocations) { + File[] indicesList = indicesLocation.listFiles(); + if (indicesList == null) { + continue; + } + for (File indexLocation : indicesList) { + indices.add(indexLocation.getName()); + } + } + return indices; + } + public Set findAllShardIds() throws Exception { if (nodeFiles == null || locks == null) { throw new ElasticSearchIllegalStateException("node is not configured to store local location"); @@ -200,7 +218,10 @@ public class NodeEnvironment extends AbstractComponent { if (!shardLocation.isDirectory()) { continue; } - shardIds.add(new ShardId(indexName, Integer.parseInt(shardLocation.getName()))); + Integer shardId = Ints.tryParse(shardLocation.getName()); + if (shardId != null) { + shardIds.add(new ShardId(indexName, shardId)); + } } } } diff --git a/src/main/java/org/elasticsearch/gateway/Gateway.java b/src/main/java/org/elasticsearch/gateway/Gateway.java index 8fdd5a84190..3e34284431c 100644 --- a/src/main/java/org/elasticsearch/gateway/Gateway.java +++ b/src/main/java/org/elasticsearch/gateway/Gateway.java @@ -39,6 +39,6 @@ public interface Gateway extends LifecycleComponent { interface GatewayStateRecoveredListener { void onSuccess(ClusterState recoveredState); - void onFailure(Throwable t); + void onFailure(String message); } } diff --git a/src/main/java/org/elasticsearch/gateway/GatewayService.java b/src/main/java/org/elasticsearch/gateway/GatewayService.java index b415cc8d33e..429b39296f4 100644 --- a/src/main/java/org/elasticsearch/gateway/GatewayService.java +++ b/src/main/java/org/elasticsearch/gateway/GatewayService.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaDataStateIndexService; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -42,7 +41,6 @@ import org.elasticsearch.discovery.DiscoveryService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; -import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; @@ -92,7 +90,8 @@ public class GatewayService extends AbstractLifecycleComponent i this.expectedNodes = componentSettings.getAsInt("expected_nodes", -1); this.recoverAfterDataNodes = componentSettings.getAsInt("recover_after_data_nodes", -1); this.expectedDataNodes = componentSettings.getAsInt("expected_data_nodes", -1); - this.recoverAfterMasterNodes = componentSettings.getAsInt("recover_after_master_nodes", -1); + // default the recover after master nodes to the minimum master nodes in the discovery + this.recoverAfterMasterNodes = componentSettings.getAsInt("recover_after_master_nodes", settings.getAsInt("discovery.zen.minimum_master_nodes", -1)); this.expectedMasterNodes = componentSettings.getAsInt("expected_master_nodes", -1); // Add the not recovered as initial state block, we don't allow anything until @@ -140,7 +139,7 @@ public class GatewayService extends AbstractLifecycleComponent i } else { logger.debug("can't wait on start for (possibly) reading state from gateway, will do it asynchronously"); } - clusterService.add(this); + clusterService.addLast(this); } @Override @@ -249,23 +248,14 @@ public class GatewayService extends AbstractLifecycleComponent i .removeGlobalBlock(STATE_NOT_RECOVERED_BLOCK); MetaData.Builder metaDataBuilder = newMetaDataBuilder() - .metaData(currentState.metaData()); - metaDataBuilder.version(recoveredState.version()); - - metaDataBuilder.persistentSettings(recoveredState.metaData().persistentSettings()); - - // add the index templates - for (Map.Entry entry : recoveredState.metaData().templates().entrySet()) { - metaDataBuilder.put(entry.getValue()); - } - + .metaData(recoveredState.metaData()); if (recoveredState.metaData().settings().getAsBoolean(MetaData.SETTING_READ_ONLY, false) || currentState.metaData().settings().getAsBoolean(MetaData.SETTING_READ_ONLY, false)) { blocks.addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK); } for (IndexMetaData indexMetaData : recoveredState.metaData()) { - metaDataBuilder.put(indexMetaData); + metaDataBuilder.put(indexMetaData, false); if (indexMetaData.state() == IndexMetaData.State.CLOSE) { blocks.addIndexBlock(indexMetaData.index(), MetaDataStateIndexService.INDEX_CLOSED_BLOCK); } @@ -276,7 +266,6 @@ public class GatewayService extends AbstractLifecycleComponent i // update the state to reflect the new metadata and routing ClusterState updatedState = newClusterStateBuilder().state(currentState) - .version(recoveredState.version()) .blocks(blocks) .metaData(metaDataBuilder) .build(); @@ -290,7 +279,8 @@ public class GatewayService extends AbstractLifecycleComponent i routingTableBuilder.add(indexRoutingBuilder); } } - routingTableBuilder.version(recoveredState.version()); + // start with 0 based versions for routing table + routingTableBuilder.version(0); // now, reroute RoutingAllocation.Result routingResult = allocationService.reroute(newClusterStateBuilder().state(updatedState).routingTable(routingTableBuilder).build()); @@ -307,9 +297,11 @@ public class GatewayService extends AbstractLifecycleComponent i } @Override - public void onFailure(Throwable t) { + public void onFailure(String message) { + recovered.set(false); + scheduledRecovery.set(false); // don't remove the block here, we don't want to allow anything in such a case - logger.error("failed recover state, blocking...", t); + logger.info("metadata state not restored, reason: {}", message); } } } diff --git a/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java b/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java index 79d8eb38034..7ddcc4d63f9 100644 --- a/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java +++ b/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java @@ -20,37 +20,29 @@ package org.elasticsearch.gateway.local; import com.google.common.collect.Sets; -import com.google.common.io.Closeables; +import gnu.trove.map.hash.TObjectIntHashMap; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.component.AbstractLifecycleComponent; -import org.elasticsearch.common.compress.lzf.LZF; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.io.FileSystemUtils; -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.io.stream.*; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.thread.LoggingRunnable; -import org.elasticsearch.common.xcontent.*; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.gateway.Gateway; import org.elasticsearch.gateway.GatewayException; +import org.elasticsearch.gateway.local.state.meta.LocalGatewayMetaState; +import org.elasticsearch.gateway.local.state.meta.TransportNodesListGatewayMetaState; import org.elasticsearch.gateway.local.state.shards.LocalGatewayShardsState; import org.elasticsearch.index.gateway.local.LocalIndexGatewayModule; -import java.io.*; import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; - -import static java.util.concurrent.Executors.newSingleThreadExecutor; -import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory; /** * @@ -62,32 +54,28 @@ public class LocalGateway extends AbstractLifecycleComponent implements private final NodeEnvironment nodeEnv; private final LocalGatewayShardsState shardsState; + private final LocalGatewayMetaState metaState; private final TransportNodesListGatewayMetaState listGatewayMetaState; - private final boolean compress; - private final boolean prettyPrint; - - private volatile LocalGatewayMetaState currentMetaState; - - private volatile ExecutorService executor; - - private volatile boolean initialized = false; - - private volatile boolean metaDataPersistedAtLeastOnce = false; + private final String initialMeta; @Inject - public LocalGateway(Settings settings, ClusterService clusterService, NodeEnvironment nodeEnv, LocalGatewayShardsState shardsState, + public LocalGateway(Settings settings, ClusterService clusterService, NodeEnvironment nodeEnv, + LocalGatewayShardsState shardsState, LocalGatewayMetaState metaState, TransportNodesListGatewayMetaState listGatewayMetaState) { super(settings); this.clusterService = clusterService; this.nodeEnv = nodeEnv; - this.listGatewayMetaState = listGatewayMetaState.initGateway(this); + this.metaState = metaState; + this.listGatewayMetaState = listGatewayMetaState; this.shardsState = shardsState; - this.compress = componentSettings.getAsBoolean("compress", true); - this.prettyPrint = componentSettings.getAsBoolean("pretty", false); + clusterService.addLast(this); + + // we define what is our minimum "master" nodes, use that to allow for recovery + this.initialMeta = componentSettings.get("initial_meta", settings.get("discovery.zen.minimum_master_nodes", "1")); } @Override @@ -95,31 +83,17 @@ public class LocalGateway extends AbstractLifecycleComponent implements return "local"; } - public LocalGatewayMetaState currentMetaState() { - lazyInitialize(); - return this.currentMetaState; - } - @Override protected void doStart() throws ElasticSearchException { - this.executor = newSingleThreadExecutor(daemonThreadFactory(settings, "gateway")); - lazyInitialize(); - clusterService.addLast(this); } @Override protected void doStop() throws ElasticSearchException { - clusterService.remove(this); - executor.shutdown(); - try { - executor.awaitTermination(10, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // ignore - } } @Override protected void doClose() throws ElasticSearchException { + clusterService.remove(this); } @Override @@ -128,32 +102,90 @@ public class LocalGateway extends AbstractLifecycleComponent implements nodesIds.addAll(clusterService.state().nodes().masterNodes().keySet()); TransportNodesListGatewayMetaState.NodesLocalGatewayMetaState nodesState = listGatewayMetaState.list(nodesIds, null).actionGet(); + + int requiredAllocation = 1; + try { + if ("quorum".equals(initialMeta)) { + if (nodesIds.size() > 2) { + requiredAllocation = (nodesIds.size() / 2) + 1; + } + } else if ("quorum-1".equals(initialMeta) || "half".equals(initialMeta)) { + if (nodesIds.size() > 2) { + requiredAllocation = ((1 + nodesIds.size()) / 2); + } + } else if ("one".equals(initialMeta)) { + requiredAllocation = 1; + } else if ("full".equals(initialMeta) || "all".equals(initialMeta)) { + requiredAllocation = nodesIds.size(); + } else if ("full-1".equals(initialMeta) || "all-1".equals(initialMeta)) { + if (nodesIds.size() > 1) { + requiredAllocation = nodesIds.size() - 1; + } + } else { + requiredAllocation = Integer.parseInt(initialMeta); + } + } catch (Exception e) { + logger.warn("failed to derived initial_meta from value {}", initialMeta); + } + if (nodesState.failures().length > 0) { for (FailedNodeException failedNodeException : nodesState.failures()) { logger.warn("failed to fetch state from node", failedNodeException); } } - TransportNodesListGatewayMetaState.NodeLocalGatewayMetaState electedState = null; + MetaData.Builder metaDataBuilder = MetaData.builder(); + TObjectIntHashMap indices = new TObjectIntHashMap(); + MetaData electedGlobalState = null; + int found = 0; for (TransportNodesListGatewayMetaState.NodeLocalGatewayMetaState nodeState : nodesState) { - if (nodeState.state() == null) { + if (nodeState.metaData() == null) { continue; } - if (electedState == null) { - electedState = nodeState; - } else if (nodeState.state().version() > electedState.state().version()) { - electedState = nodeState; + found++; + if (electedGlobalState == null) { + electedGlobalState = nodeState.metaData(); + } else if (nodeState.metaData().version() > electedGlobalState.version()) { + electedGlobalState = nodeState.metaData(); + } + for (IndexMetaData indexMetaData : nodeState.metaData().indices().values()) { + indices.adjustOrPutValue(indexMetaData.index(), 1, 1); } } - if (electedState == null) { - logger.debug("no state elected"); - listener.onSuccess(ClusterState.builder().build()); - } else { - logger.debug("elected state from [{}]", electedState.node()); - ClusterState.Builder builder = ClusterState.builder().version(electedState.state().version()); - builder.metaData(MetaData.builder().metaData(electedState.state().metaData()).version(electedState.state().version())); - listener.onSuccess(builder.build()); + if (found < requiredAllocation) { + listener.onFailure("found [" + found + "] metadata states, required [" + requiredAllocation + "]"); + return; } + // update the global state, and clean the indices, we elect them in the next phase + metaDataBuilder.metaData(electedGlobalState).removeAllIndices(); + for (String index : indices.keySet()) { + IndexMetaData electedIndexMetaData = null; + int indexMetaDataCount = 0; + for (TransportNodesListGatewayMetaState.NodeLocalGatewayMetaState nodeState : nodesState) { + if (nodeState.metaData() == null) { + continue; + } + IndexMetaData indexMetaData = nodeState.metaData().index(index); + if (indexMetaData == null) { + continue; + } + if (electedIndexMetaData == null) { + electedIndexMetaData = indexMetaData; + } else if (indexMetaData.version() > electedIndexMetaData.version()) { + electedIndexMetaData = indexMetaData; + } + indexMetaDataCount++; + } + if (electedIndexMetaData != null) { + if (indexMetaDataCount < requiredAllocation) { + logger.debug("[{}] found [{}], required [{}], not adding", index, indexMetaDataCount, requiredAllocation); + } + metaDataBuilder.put(electedIndexMetaData, false); + } + } + ClusterState.Builder builder = ClusterState.builder(); + builder.metaData(metaDataBuilder); + listener.onSuccess(builder.build()); } @Override @@ -172,187 +204,7 @@ public class LocalGateway extends AbstractLifecycleComponent implements if (event.state().blocks().disableStatePersistence()) { return; } - - // we only write the local metadata if this is a possible master node - if (event.state().nodes().localNode().masterNode() && (event.metaDataChanged() || !metaDataPersistedAtLeastOnce)) { - executor.execute(new LoggingRunnable(logger, new PersistMetaData(event))); - } - + metaState.clusterChanged(event); shardsState.clusterChanged(event); } - - /** - * We do here lazy initialization on not only on start(), since we might be called before start by another node (really will - * happen in term of timing in testing, but still), and we want to return the cluster state when we can. - *

    - * It is synchronized since we want to wait for it to be loaded if called concurrently. There should really be a nicer - * solution here, but for now, its good enough. - */ - private synchronized void lazyInitialize() { - if (initialized) { - return; - } - initialized = true; - - if (clusterService.localNode().masterNode()) { - try { - File latest = findLatestMetaStateVersion(); - if (latest != null) { - logger.debug("[find_latest_state]: loading metadata from [{}]", latest.getAbsolutePath()); - this.currentMetaState = readMetaState(Streams.copyToByteArray(new FileInputStream(latest))); - } else { - logger.debug("[find_latest_state]: no metadata state loaded"); - } - } catch (Exception e) { - logger.warn("failed to read local state (metadata)", e); - } - } - } - - private File findLatestMetaStateVersion() throws IOException { - long index = -1; - File latest = null; - for (File dataLocation : nodeEnv.nodeDataLocations()) { - File stateLocation = new File(dataLocation, "_state"); - if (!stateLocation.exists()) { - continue; - } - File[] stateFiles = stateLocation.listFiles(); - if (stateFiles == null) { - continue; - } - for (File stateFile : stateFiles) { - if (logger.isTraceEnabled()) { - logger.trace("[find_latest_state]: processing [" + stateFile.getName() + "]"); - } - String name = stateFile.getName(); - if (!name.startsWith("metadata-")) { - continue; - } - long fileIndex = Long.parseLong(name.substring(name.indexOf('-') + 1)); - if (fileIndex >= index) { - // try and read the meta data - try { - byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile)); - if (data.length == 0) { - logger.debug("[find_latest_state]: not data for [" + name + "], ignoring..."); - continue; - } - readMetaState(data); - index = fileIndex; - latest = stateFile; - } catch (IOException e) { - logger.warn("[find_latest_state]: failed to read state from [" + name + "], ignoring...", e); - } - } - } - } - return latest; - } - - private LocalGatewayMetaState readMetaState(byte[] data) throws IOException { - XContentParser parser = null; - try { - if (LZF.isCompressed(data)) { - BytesStreamInput siBytes = new BytesStreamInput(data, false); - LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes); - parser = XContentFactory.xContent(XContentType.JSON).createParser(siLzf); - } else { - parser = XContentFactory.xContent(XContentType.JSON).createParser(data); - } - return LocalGatewayMetaState.Builder.fromXContent(parser); - } finally { - if (parser != null) { - parser.close(); - } - } - } - - class PersistMetaData implements Runnable { - private final ClusterChangedEvent event; - - public PersistMetaData(ClusterChangedEvent event) { - this.event = event; - } - - @Override - public void run() { - LocalGatewayMetaState.Builder builder = LocalGatewayMetaState.builder(); - if (currentMetaState != null) { - builder.state(currentMetaState); - } - final long version = event.state().metaData().version(); - builder.version(version); - builder.metaData(event.state().metaData()); - LocalGatewayMetaState stateToWrite = builder.build(); - - CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry(); - StreamOutput streamOutput; - try { - try { - if (compress) { - streamOutput = cachedEntry.cachedLZFBytes(); - } else { - streamOutput = cachedEntry.cachedBytes(); - } - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON, streamOutput); - if (prettyPrint) { - xContentBuilder.prettyPrint(); - } - xContentBuilder.startObject(); - LocalGatewayMetaState.Builder.toXContent(stateToWrite, xContentBuilder, ToXContent.EMPTY_PARAMS); - xContentBuilder.endObject(); - xContentBuilder.close(); - } catch (Exception e) { - logger.warn("failed to serialize local gateway state", e); - return; - } - - boolean serializedAtLeastOnce = false; - for (File dataLocation : nodeEnv.nodeDataLocations()) { - File stateLocation = new File(dataLocation, "_state"); - if (!stateLocation.exists()) { - FileSystemUtils.mkdirs(stateLocation); - } - File stateFile = new File(stateLocation, "metadata-" + version); - FileOutputStream fos = null; - try { - fos = new FileOutputStream(stateFile); - fos.write(cachedEntry.bytes().underlyingBytes(), 0, cachedEntry.bytes().size()); - fos.getChannel().force(true); - serializedAtLeastOnce = true; - } catch (Exception e) { - logger.warn("failed to write local gateway state to {}", e, stateFile); - } finally { - Closeables.closeQuietly(fos); - } - } - if (serializedAtLeastOnce) { - currentMetaState = stateToWrite; - metaDataPersistedAtLeastOnce = true; - - // delete all the other files - for (File dataLocation : nodeEnv.nodeDataLocations()) { - File stateLocation = new File(dataLocation, "_state"); - if (!stateLocation.exists()) { - continue; - } - File[] files = stateLocation.listFiles(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.startsWith("metadata-") && !name.equals("metadata-" + version); - } - }); - if (files != null) { - for (File file : files) { - file.delete(); - } - } - } - } - } finally { - CachedStreamOutput.pushEntry(cachedEntry); - } - } - } } diff --git a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayMetaState.java b/src/main/java/org/elasticsearch/gateway/local/LocalGatewayMetaState.java deleted file mode 100644 index 92a26d15303..00000000000 --- a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayMetaState.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.gateway.local; - -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; - -import java.io.IOException; - -/** - * - */ -public class LocalGatewayMetaState { - - private final long version; - - private final MetaData metaData; - - public LocalGatewayMetaState(long version, MetaData metaData) { - this.version = version; - this.metaData = metaData; - } - - public long version() { - return version; - } - - public MetaData metaData() { - return metaData; - } - - public static Builder builder() { - return new Builder(); - } - - public static class Builder { - - private long version; - - private MetaData metaData; - - public Builder state(LocalGatewayMetaState state) { - this.version = state.version(); - this.metaData = state.metaData(); - return this; - } - - public Builder version(long version) { - this.version = version; - return this; - } - - public Builder metaData(MetaData metaData) { - this.metaData = metaData; - return this; - } - - public LocalGatewayMetaState build() { - return new LocalGatewayMetaState(version, metaData); - } - - public static void toXContent(LocalGatewayMetaState state, XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startObject("state"); - - builder.field("version", state.version()); - MetaData.Builder.toXContent(state.metaData(), builder, params); - - builder.endObject(); - } - - public static LocalGatewayMetaState fromXContent(XContentParser parser) throws IOException { - Builder builder = new Builder(); - - String currentFieldName = null; - XContentParser.Token token = parser.nextToken(); - if (token == null) { - // no data... - return builder.build(); - } - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - if ("meta-data".equals(currentFieldName)) { - builder.metaData = MetaData.Builder.fromXContent(parser); - } - } else if (token.isValue()) { - if ("version".equals(currentFieldName)) { - builder.version = parser.longValue(); - } - } - } - - return builder.build(); - } - - public static LocalGatewayMetaState readFrom(StreamInput in) throws IOException { - LocalGatewayMetaState.Builder builder = new Builder(); - builder.version = in.readLong(); - builder.metaData = MetaData.Builder.readFrom(in); - return builder.build(); - } - - public static void writeTo(LocalGatewayMetaState state, StreamOutput out) throws IOException { - out.writeLong(state.version()); - MetaData.Builder.writeTo(state.metaData(), out); - } - } - -} diff --git a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayModule.java b/src/main/java/org/elasticsearch/gateway/local/LocalGatewayModule.java index c2f2b727950..36f8465f36a 100644 --- a/src/main/java/org/elasticsearch/gateway/local/LocalGatewayModule.java +++ b/src/main/java/org/elasticsearch/gateway/local/LocalGatewayModule.java @@ -24,6 +24,8 @@ import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.inject.PreProcessModule; import org.elasticsearch.gateway.Gateway; +import org.elasticsearch.gateway.local.state.meta.LocalGatewayMetaState; +import org.elasticsearch.gateway.local.state.meta.TransportNodesListGatewayMetaState; import org.elasticsearch.gateway.local.state.shards.LocalGatewayShardsState; import org.elasticsearch.gateway.local.state.shards.TransportNodesListGatewayStartedShards; @@ -37,6 +39,7 @@ public class LocalGatewayModule extends AbstractModule implements PreProcessModu bind(Gateway.class).to(LocalGateway.class).asEagerSingleton(); bind(LocalGatewayShardsState.class).asEagerSingleton(); bind(TransportNodesListGatewayMetaState.class).asEagerSingleton(); + bind(LocalGatewayMetaState.class).asEagerSingleton(); bind(TransportNodesListGatewayStartedShards.class).asEagerSingleton(); } diff --git a/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java new file mode 100644 index 00000000000..dab24bd85d8 --- /dev/null +++ b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java @@ -0,0 +1,454 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gateway.local.state.meta; + +import com.google.common.io.Closeables; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.io.stream.CachedStreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.index.Index; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Set; + +/** + * + */ +public class LocalGatewayMetaState extends AbstractComponent implements ClusterStateListener { + + private final NodeEnvironment nodeEnv; + + private volatile MetaData currentMetaData; + + @Inject + public LocalGatewayMetaState(Settings settings, NodeEnvironment nodeEnv, TransportNodesListGatewayMetaState nodesListGatewayMetaState) throws Exception { + super(settings); + this.nodeEnv = nodeEnv; + + nodesListGatewayMetaState.init(this); + + try { + pre019Upgrade(); + long start = System.currentTimeMillis(); + loadState(); + logger.debug("took {} to load state", TimeValue.timeValueMillis(System.currentTimeMillis() - start)); + } catch (Exception e) { + logger.error("failed to read local state, exiting...", e); + throw e; + } + } + + public MetaData currentMetaData() { + return currentMetaData; + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (event.state().blocks().disableStatePersistence()) { + return; + } + + if (!event.state().nodes().localNode().masterNode()) { + return; + } + + if (!event.metaDataChanged()) { + return; + } + + // check if the global state changed? + boolean success = true; + if (currentMetaData == null || !MetaData.isGlobalStateEquals(currentMetaData, event.state().metaData())) { + try { + writeGlobalState("changed", event.state().metaData(), currentMetaData); + } catch (Exception e) { + success = false; + } + } + + // check and write changes in indices + for (IndexMetaData indexMetaData : event.state().metaData()) { + String writeReason = null; + IndexMetaData currentIndexMetaData = currentMetaData == null ? null : currentMetaData.index(indexMetaData.index()); + if (currentIndexMetaData == null) { + writeReason = "freshly created"; + } else if (currentIndexMetaData.version() != indexMetaData.version()) { + writeReason = "version changed from [" + currentIndexMetaData.version() + "] to [" + indexMetaData.version() + "]"; + } + + // we update the writeReason only if we really need to write it + if (writeReason == null) { + continue; + } + + try { + writeIndex(writeReason, indexMetaData, currentIndexMetaData); + } catch (Exception e) { + success = false; + } + } + + // delete indices that are no longer there... + if (currentMetaData != null) { + for (IndexMetaData current : currentMetaData) { + if (event.state().metaData().index(current.index()) == null) { + deleteIndex(current.index()); + } + } + } + + if (success) { + currentMetaData = event.state().metaData(); + } + } + + private void deleteIndex(String index) { + logger.trace("[{}] delete index state", index); + File[] indexLocations = nodeEnv.indexLocations(new Index(index)); + for (File indexLocation : indexLocations) { + if (!indexLocation.exists()) { + continue; + } + FileSystemUtils.deleteRecursively(new File(indexLocation, "_state")); + } + } + + private void writeIndex(String reason, IndexMetaData indexMetaData, @Nullable IndexMetaData previousIndexMetaData) throws Exception { + logger.trace("[{}] writing state, reason [{}]", indexMetaData.index(), reason); + CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry(); + try { + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, cachedEntry.cachedBytes()); + builder.startObject(); + IndexMetaData.Builder.toXContent(indexMetaData, builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + builder.flush(); + + Exception lastFailure = null; + boolean wroteAtLeastOnce = false; + for (File indexLocation : nodeEnv.indexLocations(new Index(indexMetaData.index()))) { + File stateLocation = new File(indexLocation, "_state"); + FileSystemUtils.mkdirs(stateLocation); + File stateFile = new File(stateLocation, "state-" + indexMetaData.version()); + + FileOutputStream fos = null; + try { + fos = new FileOutputStream(stateFile); + fos.write(cachedEntry.bytes().underlyingBytes(), 0, cachedEntry.bytes().size()); + fos.getChannel().force(true); + Closeables.closeQuietly(fos); + wroteAtLeastOnce = true; + } catch (Exception e) { + lastFailure = e; + } finally { + Closeables.closeQuietly(fos); + } + } + + if (!wroteAtLeastOnce) { + logger.warn("[{}]: failed to state", lastFailure, indexMetaData.index()); + throw new IOException("failed to write state for [" + indexMetaData.index() + "]", lastFailure); + } + + // delete the old files + if (previousIndexMetaData != null && previousIndexMetaData.version() != indexMetaData.version()) { + for (File indexLocation : nodeEnv.indexLocations(new Index(indexMetaData.index()))) { + File stateFile = new File(new File(indexLocation, "_state"), "state-" + previousIndexMetaData.version()); + stateFile.delete(); + } + } + } finally { + CachedStreamOutput.pushEntry(cachedEntry); + } + } + + private void writeGlobalState(String reason, MetaData metaData, @Nullable MetaData previousMetaData) throws Exception { + logger.trace("[_global] writing state, reason [{}]", reason); + // create metadata to write with just the global state + MetaData globalMetaData = MetaData.builder().metaData(metaData).removeAllIndices().build(); + + CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry(); + try { + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, cachedEntry.cachedBytes()); + builder.startObject(); + MetaData.Builder.toXContent(globalMetaData, builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + builder.flush(); + + Exception lastFailure = null; + boolean wroteAtLeastOnce = false; + for (File dataLocation : nodeEnv.nodeDataLocations()) { + File stateLocation = new File(dataLocation, "_state"); + FileSystemUtils.mkdirs(stateLocation); + File stateFile = new File(stateLocation, "global-" + globalMetaData.version()); + + FileOutputStream fos = null; + try { + fos = new FileOutputStream(stateFile); + fos.write(cachedEntry.bytes().underlyingBytes(), 0, cachedEntry.bytes().size()); + fos.getChannel().force(true); + Closeables.closeQuietly(fos); + wroteAtLeastOnce = true; + } catch (Exception e) { + lastFailure = e; + } finally { + Closeables.closeQuietly(fos); + } + } + + if (!wroteAtLeastOnce) { + logger.warn("[_global]: failed to write global state", lastFailure); + throw new IOException("failed to write global state", lastFailure); + } + + // delete the old files + if (previousMetaData != null && previousMetaData.version() != currentMetaData.version()) { + for (File dataLocation : nodeEnv.nodeDataLocations()) { + File stateFile = new File(new File(dataLocation, "_state"), "global-" + previousMetaData.version()); + stateFile.delete(); + } + } + } finally { + CachedStreamOutput.pushEntry(cachedEntry); + } + } + + private void loadState() throws Exception { + MetaData.Builder metaDataBuilder = MetaData.builder(); + MetaData globalMetaData = loadGlobalState(); + if (globalMetaData != null) { + metaDataBuilder.metaData(globalMetaData); + } + + Set indices = nodeEnv.finalAllIndices(); + for (String index : indices) { + IndexMetaData indexMetaData = loadIndex(index); + if (indexMetaData == null) { + logger.debug("[{}] failed to find metadata for existing index location", index); + } else { + metaDataBuilder.put(indexMetaData, false); + } + } + currentMetaData = metaDataBuilder.build(); + } + + private IndexMetaData loadIndex(String index) { + long highestVersion = -1; + IndexMetaData indexMetaData = null; + for (File indexLocation : nodeEnv.indexLocations(new Index(index))) { + File stateDir = new File(indexLocation, "_state"); + if (!stateDir.exists() || !stateDir.isDirectory()) { + continue; + } + // now, iterate over the current versions, and find latest one + File[] stateFiles = stateDir.listFiles(); + if (stateFiles == null) { + continue; + } + for (File stateFile : stateFiles) { + if (!stateFile.getName().startsWith("state-")) { + continue; + } + try { + long version = Long.parseLong(stateFile.getName().substring("state-".length())); + if (version > highestVersion) { + byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile)); + if (data.length == 0) { + logger.debug("[{}]: no data for [" + stateFile.getAbsolutePath() + "], ignoring...", index); + continue; + } + XContentParser parser = null; + try { + parser = XContentHelper.createParser(data, 0, data.length); + parser.nextToken(); // move to START_OBJECT + indexMetaData = IndexMetaData.Builder.fromXContent(parser); + highestVersion = version; + } finally { + if (parser != null) { + parser.close(); + } + } + } + } catch (Exception e) { + logger.debug("[{}]: failed to read [" + stateFile.getAbsolutePath() + "], ignoring...", e, index); + } + } + } + return indexMetaData; + } + + private MetaData loadGlobalState() { + long highestVersion = -1; + MetaData metaData = null; + for (File dataLocation : nodeEnv.nodeDataLocations()) { + File stateLocation = new File(dataLocation, "_state"); + if (!stateLocation.exists()) { + continue; + } + File[] stateFiles = stateLocation.listFiles(); + if (stateFiles == null) { + continue; + } + for (File stateFile : stateFiles) { + String name = stateFile.getName(); + if (!name.startsWith("global-")) { + continue; + } + try { + long version = Long.parseLong(stateFile.getName().substring("global-".length())); + if (version > highestVersion) { + byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile)); + if (data.length == 0) { + logger.debug("[_global] no data for [" + stateFile.getAbsolutePath() + "], ignoring..."); + continue; + } + + XContentParser parser = null; + try { + parser = XContentHelper.createParser(data, 0, data.length); + metaData = MetaData.Builder.fromXContent(parser); + highestVersion = version; + } finally { + if (parser != null) { + parser.close(); + } + } + } + } catch (Exception e) { + logger.debug(""); + } + } + } + + return metaData; + } + + private void pre019Upgrade() throws Exception { + long index = -1; + File metaDataFile = null; + MetaData metaData = null; + long version = -1; + for (File dataLocation : nodeEnv.nodeDataLocations()) { + File stateLocation = new File(dataLocation, "_state"); + if (!stateLocation.exists()) { + continue; + } + File[] stateFiles = stateLocation.listFiles(); + if (stateFiles == null) { + continue; + } + for (File stateFile : stateFiles) { + if (logger.isTraceEnabled()) { + logger.trace("[upgrade]: processing [" + stateFile.getName() + "]"); + } + String name = stateFile.getName(); + if (!name.startsWith("metadata-")) { + continue; + } + long fileIndex = Long.parseLong(name.substring(name.indexOf('-') + 1)); + if (fileIndex >= index) { + // try and read the meta data + try { + byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile)); + if (data.length == 0) { + continue; + } + XContentParser parser = XContentHelper.createParser(data, 0, data.length); + try { + String currentFieldName = null; + XContentParser.Token token = parser.nextToken(); + if (token != null) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if ("meta-data".equals(currentFieldName)) { + metaData = MetaData.Builder.fromXContent(parser); + } + } else if (token.isValue()) { + if ("version".equals(currentFieldName)) { + version = parser.longValue(); + } + } + } + } + } finally { + parser.close(); + } + index = fileIndex; + metaDataFile = stateFile; + } catch (IOException e) { + logger.warn("failed to read pre 0.19 state from [" + name + "], ignoring...", e); + } + } + } + } + if (metaData == null) { + return; + } + + logger.info("found old metadata state, loading metadata from [{}] and converting to new metadata location and strucutre...", metaDataFile.getAbsolutePath()); + + writeGlobalState("upgrade", MetaData.builder().metaData(metaData).version(version).build(), null); + for (IndexMetaData indexMetaData : metaData) { + writeIndex("upgrade", IndexMetaData.newIndexMetaDataBuilder(indexMetaData).version(version).build(), null); + } + + // rename shards state to backup state + File backupFile = new File(metaDataFile.getParentFile(), "backup-" + metaDataFile.getName()); + if (!metaDataFile.renameTo(backupFile)) { + throw new IOException("failed to rename old state to backup state [" + metaDataFile.getAbsolutePath() + "]"); + } + + // delete all other shards state files + for (File dataLocation : nodeEnv.nodeDataLocations()) { + File stateLocation = new File(dataLocation, "_state"); + if (!stateLocation.exists()) { + continue; + } + File[] stateFiles = stateLocation.listFiles(); + if (stateFiles == null) { + continue; + } + for (File stateFile : stateFiles) { + String name = stateFile.getName(); + if (!name.startsWith("metadata-")) { + continue; + } + stateFile.delete(); + } + } + + logger.info("conversion to new metadata location and format done, backup create at [{}]", backupFile.getAbsolutePath()); + } +} diff --git a/src/main/java/org/elasticsearch/gateway/local/TransportNodesListGatewayMetaState.java b/src/main/java/org/elasticsearch/gateway/local/state/meta/TransportNodesListGatewayMetaState.java similarity index 91% rename from src/main/java/org/elasticsearch/gateway/local/TransportNodesListGatewayMetaState.java rename to src/main/java/org/elasticsearch/gateway/local/state/meta/TransportNodesListGatewayMetaState.java index 13d57974de8..e798b03bff5 100644 --- a/src/main/java/org/elasticsearch/gateway/local/TransportNodesListGatewayMetaState.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/meta/TransportNodesListGatewayMetaState.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.gateway.local; +package org.elasticsearch.gateway.local.state.meta; import com.google.common.collect.Lists; import org.elasticsearch.ElasticSearchException; @@ -26,6 +26,7 @@ import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.nodes.*; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Inject; @@ -46,15 +47,15 @@ import java.util.concurrent.atomic.AtomicReferenceArray; */ public class TransportNodesListGatewayMetaState extends TransportNodesOperationAction { - private LocalGateway gateway; + private LocalGatewayMetaState metaState; @Inject public TransportNodesListGatewayMetaState(Settings settings, ClusterName clusterName, ThreadPool threadPool, ClusterService clusterService, TransportService transportService) { super(settings, clusterName, threadPool, clusterService, transportService); } - TransportNodesListGatewayMetaState initGateway(LocalGateway gateway) { - this.gateway = gateway; + TransportNodesListGatewayMetaState init(LocalGatewayMetaState metaState) { + this.metaState = metaState; return this; } @@ -115,7 +116,7 @@ public class TransportNodesListGatewayMetaState extends TransportNodesOperationA @Override protected NodeLocalGatewayMetaState nodeOperation(NodeRequest request) throws ElasticSearchException { - return new NodeLocalGatewayMetaState(clusterService.localNode(), gateway.currentMetaState()); + return new NodeLocalGatewayMetaState(clusterService.localNode(), metaState.currentMetaData()); } @Override @@ -208,36 +209,36 @@ public class TransportNodesListGatewayMetaState extends TransportNodesOperationA public static class NodeLocalGatewayMetaState extends NodeOperationResponse { - private LocalGatewayMetaState state; + private MetaData metaData; NodeLocalGatewayMetaState() { } - public NodeLocalGatewayMetaState(DiscoveryNode node, LocalGatewayMetaState state) { + public NodeLocalGatewayMetaState(DiscoveryNode node, MetaData metaData) { super(node); - this.state = state; + this.metaData = metaData; } - public LocalGatewayMetaState state() { - return state; + public MetaData metaData() { + return metaData; } @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); if (in.readBoolean()) { - state = LocalGatewayMetaState.Builder.readFrom(in); + metaData = MetaData.Builder.readFrom(in); } } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (state == null) { + if (metaData == null) { out.writeBoolean(false); } else { out.writeBoolean(true); - LocalGatewayMetaState.Builder.writeTo(state, out); + MetaData.Builder.writeTo(metaData, out); } } } diff --git a/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java b/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java index 8b7cdee383b..194548faac5 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java @@ -22,7 +22,6 @@ package org.elasticsearch.gateway.local.state.shards; import com.google.common.collect.Maps; import com.google.common.io.Closeables; import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.routing.*; import org.elasticsearch.common.Nullable; @@ -37,10 +36,7 @@ import org.elasticsearch.common.io.stream.CachedStreamOutput; import org.elasticsearch.common.io.stream.LZFStreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.*; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.shard.ShardId; @@ -57,21 +53,27 @@ import java.util.Set; public class LocalGatewayShardsState extends AbstractComponent implements ClusterStateListener { private final NodeEnvironment nodeEnv; - private final ClusterService clusterService; - private volatile boolean initialized = false; private volatile Map currentState = Maps.newHashMap(); @Inject - public LocalGatewayShardsState(Settings settings, NodeEnvironment nodeEnv, ClusterService clusterService, TransportNodesListGatewayStartedShards listGatewayStartedShards) { + public LocalGatewayShardsState(Settings settings, NodeEnvironment nodeEnv, TransportNodesListGatewayStartedShards listGatewayStartedShards) throws Exception { super(settings); this.nodeEnv = nodeEnv; - this.clusterService = clusterService; listGatewayStartedShards.initGateway(this); + + try { + pre019Upgrade(); + long start = System.currentTimeMillis(); + loadStartedShards(); + logger.debug("took {} to load started shards state", TimeValue.timeValueMillis(System.currentTimeMillis() - start)); + } catch (Exception e) { + logger.error("failed to read local state (started shards), exiting...", e); + throw e; + } } public Map currentStartedShards() { - lazyInitialize(); return this.currentState; } @@ -160,30 +162,6 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste this.currentState = newState; } - private synchronized void lazyInitialize() { - if (initialized) { - return; - } - initialized = true; - - // we only persist shards state for data nodes - if (!clusterService.localNode().dataNode()) { - return; - } - - try { - pre019Upgrade(); - long start = System.currentTimeMillis(); - loadStartedShards(); - logger.debug("took {} to load started shards state", TimeValue.timeValueMillis(System.currentTimeMillis() - start)); - } catch (Exception e) { - logger.error("failed to read local state (started shards), exiting...", e); - // ugly, but, if we fail to read it, bail completely so we don't have any node corrupting the cluster - System.exit(1); - } - } - - private void loadStartedShards() throws Exception { Set shardIds = nodeEnv.findAllShardIds(); long highestVersion = -1; @@ -244,13 +222,7 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste private long readShardState(byte[] data) throws Exception { XContentParser parser = null; try { - if (LZF.isCompressed(data)) { - BytesStreamInput siBytes = new BytesStreamInput(data, false); - LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes); - parser = XContentFactory.xContent(XContentType.JSON).createParser(siLzf); - } else { - parser = XContentFactory.xContent(XContentType.JSON).createParser(data); - } + parser = XContentHelper.createParser(data, 0, data.length); XContentParser.Token token = parser.nextToken(); if (token == null) { return -1; @@ -313,7 +285,7 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste } // delete the old files - if (previousStateInfo != null) { + if (previousStateInfo != null && previousStateInfo.version != shardStateInfo.version) { for (File shardLocation : nodeEnv.shardLocations(shardId)) { File stateFile = new File(new File(shardLocation, "_state"), "state-" + previousStateInfo.version); stateFile.delete(); @@ -361,13 +333,13 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste try { byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile)); if (data.length == 0) { - logger.debug("[find_latest_state]: not data for [" + name + "], ignoring..."); + logger.debug("[upgrade]: not data for [" + name + "], ignoring..."); } pre09ReadState(data); index = fileIndex; latest = stateFile; } catch (IOException e) { - logger.warn("[find_latest_state]: failed to read state from [" + name + "], ignoring...", e); + logger.warn("[upgrade]: failed to read state from [" + name + "], ignoring...", e); } } } @@ -400,9 +372,6 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste continue; } for (File stateFile : stateFiles) { - if (logger.isTraceEnabled()) { - logger.trace("[find_latest_state]: processing [" + stateFile.getName() + "]"); - } String name = stateFile.getName(); if (!name.startsWith("shards-")) { continue; diff --git a/src/main/java/org/elasticsearch/gateway/shared/SharedStorageGateway.java b/src/main/java/org/elasticsearch/gateway/shared/SharedStorageGateway.java index e2f121d946e..3b3d92cbfe0 100644 --- a/src/main/java/org/elasticsearch/gateway/shared/SharedStorageGateway.java +++ b/src/main/java/org/elasticsearch/gateway/shared/SharedStorageGateway.java @@ -20,6 +20,7 @@ package org.elasticsearch.gateway.shared; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -53,16 +54,20 @@ public abstract class SharedStorageGateway extends AbstractLifecycleComponent starting the previous master node again..."); startNode(masterNodeName, settings); - clusterHealthResponse = client("node1").admin().cluster().prepareHealth().setWaitForNodes("2").execute().actionGet(); + clusterHealthResponse = client("node1").admin().cluster().prepareHealth().setWaitForYellowStatus().setWaitForNodes("2").execute().actionGet(); assertThat(clusterHealthResponse.timedOut(), equalTo(false)); state = client("node1").admin().cluster().prepareState().setLocal(true).execute().actionGet().state(); From f320cf450977ae94e200c85e6a710a9b518966c0 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 22 Jan 2012 23:57:02 +0200 Subject: [PATCH 192/270] REST: `/` to properly return error code (HEAD and GET) when blocked (not recovered, no master), as well as cluster health (red == 503), closes #1633. --- .../health/TransportClusterHealthAction.java | 4 +- .../cluster/state/ClusterStateRequest.java | 10 + .../state/ClusterStateRequestBuilder.java | 18 + .../cluster/block/ClusterBlocks.java | 13 + .../health/RestClusterHealthAction.java | 6 +- .../rest/action/main/RestMainAction.java | 107 ++- .../rest/action/main/quotes.json | 653 ------------------ 7 files changed, 92 insertions(+), 719 deletions(-) delete mode 100644 src/main/java/org/elasticsearch/rest/action/main/quotes.json diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java index 5090aa77690..b3c1732c8e3 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java @@ -31,8 +31,8 @@ import org.elasticsearch.cluster.routing.RoutingTableValidation; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.indices.IndexMissingException; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -266,7 +266,7 @@ public class TransportClusterHealthAction extends TransportMasterNodeOperationAc response.status = ClusterHealthStatus.GREEN; if (!response.validationFailures().isEmpty()) { response.status = ClusterHealthStatus.RED; - } else if (clusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { + } else if (clusterState.blocks().hasGlobalBlock(RestStatus.SERVICE_UNAVAILABLE)) { response.status = ClusterHealthStatus.RED; } else { for (ClusterIndexHealth indexHealth : response) { diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java b/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java index a9a94302413..c4905db1ef7 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java @@ -54,6 +54,16 @@ public class ClusterStateRequest extends MasterNodeOperationRequest { return null; } + public ClusterStateRequest filterAll() { + filterRoutingTable = true; + filterNodes = true; + filterMetaData = true; + filterBlocks = true; + filteredIndices = Strings.EMPTY_ARRAY; + filteredIndexTemplates = Strings.EMPTY_ARRAY; + return this; + } + public boolean filterRoutingTable() { return filterRoutingTable; } diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java index bf5f997bd8b..83aaa234393 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java @@ -33,6 +33,19 @@ public class ClusterStateRequestBuilder extends BaseClusterRequestBuilderfalse. @@ -69,6 +82,11 @@ public class ClusterStateRequestBuilder extends BaseClusterRequestBuilder rootNode; - - private final int quotesSize; - @Inject public RestMainAction(Settings settings, Client client, RestController controller) { super(settings, client); - Map rootNode; - int quotesSize; - try { - XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(Classes.getDefaultClassLoader().getResourceAsStream("org/elasticsearch/rest/action/main/quotes.json")); - rootNode = parser.map(); - List arrayNode = (List) rootNode.get("quotes"); - quotesSize = arrayNode.size(); - } catch (Exception e) { - rootNode = null; - quotesSize = -1; - } - this.rootNode = rootNode; - this.quotesSize = quotesSize; controller.registerHandler(GET, "/", this); controller.registerHandler(HEAD, "/", this); } @Override - public void handleRequest(RestRequest request, RestChannel channel) { - try { - if (request.method() == RestRequest.Method.HEAD) { - channel.sendResponse(new StringRestResponse(RestStatus.OK)); - return; - } - XContentBuilder builder = RestXContentBuilder.restContentBuilder(request).prettyPrint(); - builder.startObject(); - builder.field("ok", true); - if (settings.get("name") != null) { - builder.field("name", settings.get("name")); - } - builder.startObject("version").field("number", Version.CURRENT.number()).field("snapshot_build", Version.CURRENT.snapshot).endObject(); - builder.field("tagline", "You Know, for Search"); - builder.field("cover", "DON'T PANIC"); - if (rootNode != null) { - builder.startObject("quote"); - List arrayNode = (List) rootNode.get("quotes"); - Map quoteNode = (Map) arrayNode.get(ThreadLocalRandom.current().nextInt(quotesSize)); - builder.field("book", quoteNode.get("book").toString()); - builder.field("chapter", quoteNode.get("chapter").toString()); - List textNodes = (List) quoteNode.get("text"); -// builder.startArray("text"); -// for (JsonNode textNode : textNodes) { -// builder.value(textNode.getValueAsText()); -// } -// builder.endArray(); - int index = 0; - for (Object textNode : textNodes) { - builder.field("text" + (++index), textNode.toString()); + public void handleRequest(final RestRequest request, final RestChannel channel) { + ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); + clusterStateRequest.masterNodeTimeout(TimeValue.timeValueMillis(0)); + clusterStateRequest.local(true); + clusterStateRequest.filterAll().filterBlocks(false); + client.admin().cluster().state(clusterStateRequest, new ActionListener() { + @Override + public void onResponse(ClusterStateResponse response) { + RestStatus status = RestStatus.OK; + if (response.state().blocks().hasGlobalBlock(RestStatus.SERVICE_UNAVAILABLE)) { + status = RestStatus.SERVICE_UNAVAILABLE; + } + if (request.method() == RestRequest.Method.HEAD) { + channel.sendResponse(new StringRestResponse(status)); + return; + } + + try { + XContentBuilder builder = RestXContentBuilder.restContentBuilder(request).prettyPrint(); + builder.startObject(); + builder.field("ok", true); + builder.field("status", status.getStatus()); + if (settings.get("name") != null) { + builder.field("name", settings.get("name")); + } + builder.startObject("version").field("number", Version.CURRENT.number()).field("snapshot_build", Version.CURRENT.snapshot).endObject(); + builder.field("tagline", "You Know, for Search"); + builder.endObject(); + channel.sendResponse(new XContentRestResponse(request, status, builder)); + } catch (Exception e) { + onFailure(e); } - builder.endObject(); } - builder.endObject(); - channel.sendResponse(new XContentRestResponse(request, RestStatus.OK, builder)); - } catch (Exception e) { - try { - channel.sendResponse(new XContentThrowableRestResponse(request, e)); - } catch (IOException e1) { - logger.warn("Failed to send response", e); + + @Override + public void onFailure(Throwable e) { + try { + channel.sendResponse(new XContentThrowableRestResponse(request, e)); + } catch (IOException e1) { + logger.warn("Failed to send response", e); + } } - } + }); } } diff --git a/src/main/java/org/elasticsearch/rest/action/main/quotes.json b/src/main/java/org/elasticsearch/rest/action/main/quotes.json deleted file mode 100644 index bd4f3de4d48..00000000000 --- a/src/main/java/org/elasticsearch/rest/action/main/quotes.json +++ /dev/null @@ -1,653 +0,0 @@ -{ - quotes:[ - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Introduction", - text:["This planet has - or rather had - a problem, which was this: most of the people living on it were unhappy for pretty much all of the time. Many solutions were suggested for this problem, but most of these were largely concerned with the movement of small green pieces of paper, which was odd because on the whole it wasn't the small green pieces of paper that were unhappy."] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Introduction", - text:["Many were increasingly of the opinion that they'd all made a big mistake in coming down from the trees in the first place. And some said that even the trees had been a bad move, and that no one should ever have left the oceans."] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Introduction", - text:["In many of the more relaxed civilizations on the Outer Eastern Rim of the Galaxy, the Hitch-Hiker's Guide has already supplanted the great Encyclopaedia Galactica as the standard repository of all knowledge and wisdom, for though it has many omissions and contains much that is apocryphal, or at least wildly inaccurate, it scores over the older, more pedestrian work in two important respects.", - "First, it is slightly cheaper; and secondly it has the words DON'T PANIC inscribed in large friendly letters on its cover." - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 1", - text:[ - "\"Some factual information for you. Have you any idea how much damage that bulldozer would suffer if I just let it roll straight over you?\"", - "\"How much?\" said Arthur.", - "\"None at all,\" said Mr Prosser." - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 1", - text:[ - "\"The mere thought,\" growled Mr. Prosser, \"hadn't even begun to speculate,\" he continued, settling himself back, \"about the merest possibility of crossing my mind.\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 2", - text:[ - "[The Guide] says that the best drink in existence is the Pan Galactic Gargle Blaster. It says that the effect of a Pan Galactic Gargle Blaster is like having your brains smashed out by a slice of lemon wrapped round a large gold brick." - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 2", - text:[ - "\"Time is an illusion. Lunchtime doubly so.\"", - "\"Very deep,\" said Arthur, \"you should send that in to the Reader's Digest. They've got a page for people like you.\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 2", - text:[ - "\"This must be Thursday,\" said Arthur to himself, sinking low over his beer, \"I never could get the hang of Thursdays.\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 3", - text:[ - "The ships hung in the sky in much the same way that bricks don't." - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 5", - text:[ - "One of the things Ford Prefect had always found hardest to understand about humans was their habit of continually stating and repeating the very very obvious, as in It's a nice day, or You're very tall, or Oh dear you seem to have fallen down a thirty-foot well, are you alright? At first Ford had formed a theory to account for this strange behaviour. If human beings don't keep exercising their lips, he thought, their mouths probably seize up. After a few months' consideration and observation he abandoned this theory in favour of a new one. If they don't keep on exercising their lips, he thought, their brains start working. After a while he abandoned this one as well as being obstructively cynical." - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 7", - text:[ - "\"You know,\" said Arthur, \"it's at times like this, when I'm trapped in a Vogon airlock with a man from Betelgeuse, and about to die of asphyxiation in deep space that I really wish I'd listened to what my mother told me when I was young.\"", - "\"Why, what did she tell you?\"", - "\"I don't know, I didn't listen.\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 8", - text:[ - "\"Space,\" it says, \"is big. Really big. You just won't believe how vastly hugely mindbogglingly big it is. I mean you may think it's a long way down the road to the chemist, but that's just peanuts to space, LISTEN!\" and so on..." - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 9", - text:[ - "Arthur looked up. \"Ford!\" he said, \"there's an infinite number of monkeys outside who want to talk to us about this script for Hamlet they've worked out.\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 9", - text:[ - "\"Ford,\" he said, \"you're turning into a penguin. Stop it\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 9", - text:[ - "\"But that's not the point!\" raged Ford \"The point is that I am now a perfectly safe penguin, and my colleague here is rapidly running out of limbs!\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 11", - text:[ - "\"Five to one against and falling...\" she said, \"four to one against and falling...three to one...two...one...probability factor of one to one...we have normality, I repeat we have normality.\" She turned her microphone off – then turned it back on, with a slight smile and continued: \"Anything you still can’t cope with is therefore your own problem.\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 11", - text:[ - "\"I think you ought to know I'm feeling very depressed,\" Marvin said." - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 11", - text:[ - "He reached out and pressed an invitingly large red button on a nearby panel. The panel lit up with the words Please do not press this button again." - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 11", - text:[ - "\"All the doors in this spaceship have a cheerful and sunny disposition. It is their pleasure to open for you, and their satisfaction to close again with the knowledge of a job well done.\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 11", - text:[ - "\"Come on,\" he droned, \"I've been ordered to take you down to the bridge. Here I am, brain the size of a planet and they ask me to take you down to the bridge. Call that job satisfaction? 'Cos I don't.\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 11", - text:[ - "\"Sorry, did I say something wrong?\" said Marvin, dragging himself on regardless. \"Pardon me for breathing, which I never do anyway so I don't know why I bother to say it, oh God I'm so depressed. Here's another one of those self-satisfied doors. Life! Don't talk to me about life.\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 12", - text:[ - "If there's anything bigger than my ego around, I want it caught and shot now." - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 13", - text:[ - "Marvin trudged on down the corridor, still moaning.", - "\"...and then of course I've got this terrible pain in all the diodes down my left hand side...\"", - "\"No?\" said Arthur grimly as he walked along beside him. \"Really?\"", - "\"Oh yes,\" said Marvin, \"I mean I've asked for them to be replaced but no one ever listens.\"", - "\"I can imagine.\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 16", - text:[ - "Isn’t it enough to see that a garden is beautiful without having to believe that there are fairies at the bottom of it too?" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 17", - text:[ - "He had found a Nutri-Matic machine which had provided him with a plastic cup filled with a liquid that was almost, but not quite, entirely unlike tea." - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 18", - text:[ - "Curiously enough, the only thing that went through the mind of the bowl of petunias as it fell was Oh no, not again. Many people have speculated that if we knew exactly why the bowl of petunias had thought that we would know a lot more about the nature of the Universe than we do now." - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 20", - text:[ - "\"Life,\" said Marvin dolefully, \"loathe it or ignore it, you can't like it.\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 23", - text:[ - "For instance, on the planet Earth, man had always assumed that he was more intelligent than dolphins because he had achieved so much - the wheel, New York, wars and so on - whilst all the dolphins had ever done was muck about in the water having a good time. But conversely, the dolphins had always believed that they were far more intelligent than man - for precisely the same reasons.", - "The last ever dolphin message was misinterpreted as a surprisingly sophisticated attempt to do a double-backwards-somersault through a hoop whilst whistling the 'Star Spangled Banner', but in fact the message was this: So long and thanks for all the fish." - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 24", - text:[ - "Looking up into the night sky is looking into infinity - distance is incomprehensible and therefore meaningless." - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 27", - text:[ - "\"Forty-two,\" said Deep Thought, with infinite majesty and calm.", - "\"The Answer to the Great Question, of Life, the Universe and Everything\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 30", - text:[ - "\"The chances of finding out what's really going on in the universe are so remote, the only thing to do is hang the sense of it and keep yourself occupied. Look at me, I design fjords. I'd far rather be happy than right any day.\"", - "\"And are you?\"", - "\"No, that's where it all falls apart I'm afraid.\"", - "\"Pity, it sounded like quite a nice lifestyle otherwise.\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 34", - text:[ - "\"What's up?\"", - "\"I don't know,\" said Marvin, \"I've never been there.\"" - ] - }, - { - book:"The Hitchhiker's Guide to the Galaxy", - chapter:"Chapter 35", - text:[ - "It said: \"The History of every major Galactic Civilization tends to pass through three distinct and recognizable phases, those of Survival, Inquiry and Sophistication, otherwise known as the How, Why and Where phases.", - "\"For instance, the first phase is characterized by the question How can we eat? the second by the question Why do we eat? and the third by the question Where shall we have lunch?\"" - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Preface", - text:[ - "There is a theory which states that if ever anyone discovers exactly what the Universe is for and why it is here, it will instantly disappear and be replaced by something even more bizarre and inexplicable.", - "There is another theory which states that this has already happened." - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 1", - text:[ - "The story so far:", - "In the beginning the Universe was created.", - "This has made a lot of people very angry and has been widely regarded as a bad move." - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 2", - text:[ - "\"Share and Enjoy\" is the company motto of the hugely successful Sirius Cybernetics Corporation Complaints division, which now covers the major land masses of three medium sized planets and is the only part of the Corporation to have shown a consistent profit in recent years." - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 2", - text:[ - "The protruding upper halves of the letters now appear, in the local language, to read \"Go stick your head in a pig\", and are no longer illuminated, except at times of special celebration." - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 3", - text:[ - "Quite how Zaphod Beeblebrox arrived at the idea of holding a seance at this point is something he was never quite clear on.", - "Obviously the subject of death was in the air, but more as something to be avoided than harped upon.", - "Possibly the horror that Zaphod experienced at the prospect of being reunited with his deceased relatives led on to the thought that they might just feel the same way about him and, what's more, be able to do something about helping to postpone this reunion." - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 3", - text:[ - "\"Concentrate,\" hissed Zaphod, \"on his name.\"", - "\"What is it?\" asked Arthur.", - "\"Zaphod Beeblebrox the Fourth.\"", - "\"What?\"", - "\"Zaphod Beeblebrox the Fourth. Concentrate!\"", - "\"The Fourth?\"", - "\"Yeah. Listen, I'm Zaphod Beeblebrox, my father was Zaphod Beeblebrox the Second, my grandfather Zaphod Beeblebrox the Third...\"", - "\"What?\"", - "\"There was an accident with a contraceptive and a time machine. Now concentrate!\"" - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 6", - text:[ - "The Guide is definitive. Reality is frequently inaccurate." - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 6", - text:[ - "\"Listen, three eyes,\" he said, \"don't you try to outweird me, I get stranger things than you free with my breakfast cereal.\"" - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 17", - text:[ - "I am the main Dish of the Day. May I interest you in parts of my body?" - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 17", - text:[ - "Shee, you guys are so unhip it's a wonder your bums don't fall off." - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 18", - text:[ - "\"The first ten million years were the worst,\" said Marvin, \"and the second ten million years, they were the worst too. The third ten million years I didn't enjoy at all. After that I went into a bit of a decline.\"" - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 18", - text:[ - "\"Er...\" he said, \"hello. Er, look, I'm sorry I'm a bit late. I've had the most ghastly time, all sorts of things cropping up at the last moment.\"", - "He seemed nervous of the expectant awed hush. He cleared his throat.", - "\"Er, how are we for time?\" he said, \"have I just got a min—\"", - "And so the Universe ended." - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 19", - text:[ - "It is known that there are an infinite number of worlds, simply because there is an infinite amount of space for them to be in. However, not every one of them is inhabited. Therefore, there must be a finite number of inhabited worlds. Any finite number divided by infinity is as near to nothing as makes no odds, so the average population of all the planets in the Universe can be said to be zero. From this it follows that the population of the whole Universe is also zero, and that any people you may meet from time to time are merely the products of a deranged imagination." - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 20", - text:[ - "The ship was rocking and swaying sickeningly as Ford and Zaphod tried to wrest control from the autopilot. The engines howled and whined like tired children in a supermarket." - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 22", - text:[ - "The trouble with most forms of transport, he thought, is basically one of them not being worth all the bother. On Earth – when there had been an Earth, before it was demolished to make way for a new hyperspace bypass – the problem had been with cars. The disadvantages involved in pulling lots of black sticky slime from out of the ground where it had been safely hidden out of harm's way, turning it into tar to cover the land with, smoke to fill the air with and pouring the rest into the sea, all seemed to outweigh the advantages of being able to get more quickly from one place to another – particularly when the place you arrived at had probably become, as a result of this, very similar to the place you had left, i.e. covered with tar, full of smoke and short of fish." - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 23", - text:[ - "The designer of the gun had clearly not been instructed to beat about the bush. \"Make it evil,\" he'd been told. \"Make it totally clear that this gun has a right end and a wrong end. Make it totally clear to anyone standing at the wrong end that things are going badly for them. If that means sticking all sort of spikes and prongs and blackened bits all over it then so be it. This is not a gun for hanging over the fireplace or sticking in the umbrella stand, it is a gun for going out and making people miserable with.\"" - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 28", - text:[ - "The major problem — one of the major problems, for there are several — one of the many major problems with governing people is that of whom you get to do it; or rather of who manages to get people to let them do it to them.", - "To summarize: it is a well known fact that those people who most want to rule people are, ipso facto, those least suited to do it. To summarize the summary: anyone who is capable of getting themselves made President should on no account be allowed to do the job. To summarize the summary of the summary: people are a problem." - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 29", - text:[ - "\"How can I tell,\" said the man, \"that the past isn't a fiction designed to account for the discrepancy between my immediate physical sensations and my state of mind?\"" - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 32", - text:[ - "\"Well, you’re obviously being totally naive of course\", said the girl, \"When you’ve been in marketing as long as I have, you'll know that before any new product can be developed it has to be properly researched. We’ve got to find out what people want from fire, how they relate to it, what sort of image it has for them.\"", - "The crowd were tense. They were expecting something wonderful from Ford.", - "\"Stick it up your nose,\" he said.", - "\"Which is precisely the sort of thing we need to know,\" insisted the girl, \"Do people want fire that can be fitted nasally?\"" - ] - }, - { - book:"The Restaurant at the End of the Universe", - chapter:"Chapter 32", - text:[ - "\"And the wheel,\" said the Captain, \"What about this wheel thingy? It sounds a terribly interesting project.\"", - "\"Ah,\" said the marketing girl, \"Well, we're having a little difficulty there.\"", - "\"Difficulty?\" exclaimed Ford. \"Difficulty? What do you mean, difficulty? It's the single simplest machine in the entire Universe!\"", - "The marketing girl soured him with a look.", - "\"Alright, Mr. Wiseguy,\" she said, \"if you're so clever, you tell us what colour it should be.\"" - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 1", - text:[ - "The regular early morning yell of horror was the sound of Arthur Dent waking up and suddenly remembering where he was." - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 1", - text:[ - "In the end, it was the Sunday afternoons he couldn't cope with, and that terrible listlessness that starts to set in about 2:55, when you know you’ve taken all the baths that you can usefully take that day, that however hard you stare at any given paragraph in the newspaper you will never actually read it, or use the revolutionary new pruning technique it describes, and that as you stare at the clock the hands will move relentlessly on to four o’clock, and you will enter the long dark teatime of the soul." - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 2", - text:[ - "\"Africa was very interesting,\" said Ford, \"I behaved very oddly there.\" [...] \"I took up being cruel to animals,\" he said airily. \"But only,\" he added, \"as a hobby.\"", - "\"Oh yes,\" said Arthur, warily.", - "\"Yes,\" Ford assured him. \"I won't disturb you with the details because they would—\"", - "\"What?\"", - "\"Disturb you. But you may be interested to know that I am singlehandedly responsible for the evolved shape of the animal you came to know in later centuries as a giraffe.\"" - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 2", - text:[ - "He gazed keenly into the distance and looked as if he would quite like the wind to blow his hair back dramatically at that point, but the wind was busy fooling around with some leaves a little way off." - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 2", - text:[ - "\"I have detected,\" he said, \"disturbances in the wash.\" [...]", - "\"The wash?\" said Arthur.", - "\"The space-time wash,\" said Ford. [...]", - "Arthur nodded, and then cleared his throat. \"Are we talking about,\" he asked cautiously, \"some sort of Vogon laundromat, or what are we talking about?\"", - "\"Eddies,\" said Ford, \"in the space-time continuum.\"", - "\"Ah,\" nodded Arthur, \"is he? Is he?\" He pushed his hands into the pocket of his dressing gown and looked knowledgeably into the distance.", - "\"What?\" said Ford.", - "\"Er, who,\" said Arthur, \"is Eddy, then, exactly, then?\"" - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 2", - text:[ - "\"There!\" said Ford, shooting out his arm. \"There, behind that sofa!\"", - "Arthur looked. Much to his surprise, there was a velvet paisley-covered Chesterfield sofa in the field in front of them. He boggled intelligently at it. Shrewd questions sprang into his mind.", - "\"Why,\" he said, \"is there a sofa in that field?\"", - "\"I told you!\" shouted Ford, leaping to his feet. \"Eddies in the space-time continuum!\"", - "\"And this is his sofa, is it?\" asked Arthur, struggling to his feet and, he hoped, though not very optimistically, to his senses." - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 6", - text:[ - "\"My doctor says that I have a malformed public-duty gland and a natural deficiency in moral fibre,\" Ford muttered to himself, \"and that I am therefore excused from saving Universes.\"" - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 9", - text:[ - "There is a moment in every dawn when light floats, there is the possibility of magic. Creation holds its breath.", - "The moment passed as it regularly did on Squornshellous Zeta, without incident." - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 9", - text:[ - "Very few things actually get manufactured these days, because in an infinitely large Universe such as, for instance, the one in which we live, most things one could possibly imagine, and a lot of things one would rather not, grow somewhere." - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 9", - text:[ - "\"My capacity for happiness,\" he added, \"you could fit into a matchbox without taking out the matches first.\" —Marvin" - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 9", - text:[ - "\"You may not instantly see why I bring the subject up, but that is because my mind works so phenomenally fast, and I am at a rough estimate thirty billion times more intelligent than you. Let me give you an example. Think of a number, any number.\"", - "\"Er, five,\" said the mattress.", - "\"Wrong,\" said Marvin. \"You see?\"", - "The mattress was much impressed by this and realized that it was in the presence of a not unremarkable mind." - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 9", - text:[ - "\"I would like to say that it is a very great pleasure, honour and privilege for me to open this bridge, but I can't because my lying circuits are all out of commission.\" —Marvin" - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 11", - text:[ - "[...] the renewed shock had nearly made him spill his drink. He drained it quickly before anything serious happened to it. He then had another quick one to follow the first one down and check that it was all right.", - "\"Freedom,\" he said aloud.", - "Trillian came on to the bridge at that point and said several enthusiastic things on the subject of freedom.", - "\"I can't cope with it,\" Zaphod said darkly, and sent a third drink down to see why the second hadn't yet reported on the condition of the first. He looked uncertainly at both of her and preferred the one on the right.", - "He poured a drink down his other throat with the plan that it would head the previous one off at the pass, join forces with it, and together they would get the second to pull itself together. Then all three would go off in search of the first, give it a good talking to and maybe a bit of a sing as well.", - "He felt uncertain as to whether the fourth drink had understood all that, so he sent down a fifth to explain the plan more fully and a sixth for moral support." - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 11", - text:[ - "There is an art, it says, or rather, a knack to flying. The knack lies in learning how to throw yourself at the ground and miss. [...] Clearly, it is this second part, the missing, which presents the difficulties." - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 11", - text:[ - "He sat up sharply and started to pull clothes on. He decided that there must be someone in the Universe feeling more wretched, miserable and forsaken than himself, and he determined to set out and find him.", - "Halfway to the bridge it occurred to him that it might be Marvin, and he returned to bed." - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 18", - text:[ - "They obstinately persisted in their absence." - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 24", - text:[ - "It is a mistake to think you can solve any major problems just with potatoes." - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 31", - text:[ - "\"That young girl,\" Marvin added unexpectedly, \"is one of the least benightedly unintelligent organic life forms it has been my profound lack of pleasure not to be able to avoid meeting.\"" - ] - }, - { - book:"Life, the Universe and Everything", - chapter:"Chapter 33", - text:[ - "He hoped and prayed that there wasn't an afterlife. Then he realized there was a contradiction involved here and merely hoped that there wasn't an afterlife." - ] - }, - { - book:"So Long And Thanks for All the Fish", - chapter:"Prologue", - text:[ - "Many were increasingly of the opinion that they'd all made a big mistake in coming down from the trees in the first place. And some said that even the trees had been a bad move, and that no one should ever have left the oceans.", - "And then, one Thursday, nearly two thousand years after one man had been nailed to a tree for saying how great it would be to be nice to people for a change, a girl sitting on her own in a small café in Rickmansworth suddenly realized what it was that had been going wrong all this time, and she finally knew how the world could be made a good and happy place. This time it was right, it would work, and no one would have to get nailed to anything.", - "Sadly, however, before she could get to a phone to tell anyone about it, the Earth was unexpectedly demolished to make way for a new hyperspace bypass, and so the idea was lost, seemingly for ever.", - "This is her story." - ] - }, - { - book:"So Long And Thanks for All the Fish", - chapter:"Chapter 21", - text:[ - "The problem is, or rather one of the problems, for there are many, a sizeable proportion of which are continually clogging up the civil, commercial, and criminal courts in all areas of the Galaxy, and especially, where possible, the more corrupt ones, this.", - "The previous sentence makes sense. That is not the problem.", - "This is:", - "Change.", - "Read it through again and you'll get it." - ] - }, - { - book:"So Long And Thanks for All the Fish", - chapter:"Chapter 23", - text:[ - "Ford: \"Life,\" he said, \"is like a grapefruit.\"", - "Creature:\"Er, how so?\"", - "Ford: \"Well, it's sort of orangey-yellow and dimpled on the outside, wet and squidgy in the middle. It's got pips inside, too. Oh, and some people have half a one for breakfast.\"" - ] - }, - { - book:"So Long And Thanks for All the Fish", - chapter:"Chapter 25", - text:[ - "\"This Arthur Dent,\" comes the cry from the furthest reaches of the galaxy, and has even now been found inscribed on a mysterious deep space probe thought to originate from an alien galaxy at a distance too hideous to contemplate, \"what is he, man or mouse? Is he interested in nothing more than tea and the wider issues of life? Has he no spirit? has he no passion? Does he not, to put it in a nutshell, fuck?\"", - "Those who wish to know should read on. Others may wish to skip on to the last chapter which is a good bit and has Marvin in it." - ] - }, - { - book:"So Long And Thanks for All the Fish", - chapter:"Chapter 31", - text:[ - "The sign said:", - "Hold stick near centre of its length. Moisten pointed end in mouth. Insert in tooth space, blunt end next to gum. Use gentle in-out motion.", - "It seemed to me,\" said Wonko the Sane, \"that any civilization that had so far lost its head as to need to include a set of detailed instructions for use in a packet of toothpicks, was no longer a civilization in which I could live and stay sane.\"" - ] - }, - { - book:"So Long And Thanks for All the Fish", - chapter:"Chapter 35", - text:[ - "The Hitchhiker's Guide to the Galaxy [...] says of the Sirius Cybernetics Corporation products that \"it is very easy to be blinded to the essential uselessness of them by the sense of achievement you get from getting them to work at all.\"" - ] - }, - { - book:"So Long And Thanks for All the Fish", - chapter:"Chapter 40", - text:[ - "\"So much time,\" it groaned, \"oh so much time. And pain as well, so much of that, and so much time to suffer it in too. One or the other on its own I could probably manage. It's the two together that really get me down.\"" - ] - }, - { - book:"So Long And Thanks for All the Fish", - chapter:"Chapter 40", - text:[ - "\"Ha!\" snapped Marvin. \"Ha!\" he repeated. \"What do you know of always? You say 'always' to me, who, because of the silly little errands your organic lifeforms keep on sending me through time on, am now thirty-seven times older than the Universe itself? Pick your words with a little more care,\" he coughed, \"and tact.\"" - ] - }, - { - book:"So Long And Thanks for All the Fish", - chapter:"Chapter 40", - text:[ - "\"We apologise for the inconvenience.\" God's Final Message to His Creation, written in letters of fire on the side of the Quentulus Quazgar Mountains.", - "\"I think,\" Marvin murmured at last, from deep within his corroding rattling thorax, \"I feel good about it.\"", - "The lights went out in his eyes for absolutely the very last time ever." - ] - }, - { - book:"So Long And Thanks for All the Fish", - chapter:"Epilogue", - text:[ - "There was a point to this story, but it has temporarily escaped the chronicler's mind." - ] - } - ] -} \ No newline at end of file From 1b7d3293073810bbbdf6579a2513313973030b06 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 23 Jan 2012 00:50:32 +0200 Subject: [PATCH 193/270] add a local gateway test to make sure we recover also latest state when updating index metadata and templates --- .../SimpleRecoveryLocalGatewayTests.java | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java b/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java index 557a2d784f2..343cabfbab3 100644 --- a/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java +++ b/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java @@ -24,7 +24,9 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.status.IndexShardStatus; import org.elasticsearch.action.admin.indices.status.IndicesStatusResponse; import org.elasticsearch.action.admin.indices.status.ShardStatus; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.gateway.Gateway; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.node.Node; @@ -39,8 +41,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.*; /** * @@ -282,6 +283,20 @@ public class SimpleRecoveryLocalGatewayTests extends AbstractNodesTests { assertThat(node2.client().prepareCount().setQuery(matchAllQuery()).execute().actionGet().count(), equalTo(3l)); } + logger.info("--> add some metadata, additional type and template"); + node2.client().admin().indices().preparePutMapping("test").setType("type2") + .setSource(jsonBuilder().startObject().startObject("type1").startObject("_source").field("enabled", false).endObject().endObject().endObject()) + .execute().actionGet(); + node2.client().admin().indices().preparePutTemplate("template_1") + .setTemplate("te*") + .setOrder(0) + .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") + .startObject("field1").field("type", "string").field("store", "yes").endObject() + .startObject("field2").field("type", "string").field("store", "yes").field("index", "not_analyzed").endObject() + .endObject().endObject().endObject()) + .execute().actionGet(); + + logger.info("--> closing the second node"); closeNode("node2"); @@ -299,6 +314,10 @@ public class SimpleRecoveryLocalGatewayTests extends AbstractNodesTests { for (int i = 0; i < 10; i++) { assertThat(node1.client().prepareCount().setQuery(matchAllQuery()).execute().actionGet().count(), equalTo(3l)); } + + ClusterState state = node1.client().admin().cluster().prepareState().execute().actionGet().state(); + assertThat(state.metaData().index("test").mapping("type2"), notNullValue()); + assertThat(state.metaData().templates().get("template_1").template(), equalTo("te*")); } @Test From 9d99c54e2ce928e613f290737b9569a44f79cf5a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 23 Jan 2012 02:04:57 +0200 Subject: [PATCH 194/270] _open and _close does not resolve aliases, closes #1628. --- .../action/admin/indices/close/TransportCloseIndexAction.java | 1 + .../action/admin/indices/open/TransportOpenIndexAction.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java b/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java index a326a775821..2d36c6b85ab 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java @@ -70,6 +70,7 @@ public class TransportCloseIndexAction extends TransportMasterNodeOperationActio @Override protected ClusterBlockException checkBlock(CloseIndexRequest request, ClusterState state) { + request.index(clusterService.state().metaData().concreteIndex(request.index())); return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA, request.index()); } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java b/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java index 3a041389bf7..bac9a962b2f 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java @@ -70,6 +70,7 @@ public class TransportOpenIndexAction extends TransportMasterNodeOperationAction @Override protected ClusterBlockException checkBlock(OpenIndexRequest request, ClusterState state) { + request.index(clusterService.state().metaData().concreteIndex(request.index())); return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA, request.index()); } From 50aad23facb03a7220f8c7e94e04baa943c1e281 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 23 Jan 2012 21:57:39 +0200 Subject: [PATCH 195/270] also store the fact that a shard is primary or not in the shard state --- .../state/shards/LocalGatewayShardsState.java | 30 ++++++++++++------- .../local/state/shards/ShardStateInfo.java | 9 +++++- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java b/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java index 194548faac5..c7ad004b93b 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java @@ -118,7 +118,7 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste // our node is not in play yet... for (MutableShardRouting shardRouting : routingNode) { if (shardRouting.active()) { - newState.put(shardRouting.shardId(), new ShardStateInfo(shardRouting.version())); + newState.put(shardRouting.shardId(), new ShardStateInfo(shardRouting.version(), shardRouting.primary())); } } } @@ -168,7 +168,7 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste Map shardsState = Maps.newHashMap(); for (ShardId shardId : shardIds) { long highestShardVersion = -1; - File highestShardFile = null; + ShardStateInfo highestShardState = null; for (File shardLocation : nodeEnv.shardLocations(shardId)) { File shardStateDir = new File(shardLocation, "_state"); if (!shardStateDir.exists() || !shardStateDir.isDirectory()) { @@ -191,10 +191,14 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste logger.debug("[{}][{}]: not data for [" + stateFile.getAbsolutePath() + "], ignoring...", shardId.index().name(), shardId.id()); continue; } - long readStateVersion = readShardState(data); - assert readStateVersion == version; + ShardStateInfo readState = readShardState(data); + if (readState == null) { + logger.debug("[{}][{}]: not data for [" + stateFile.getAbsolutePath() + "], ignoring...", shardId.index().name(), shardId.id()); + continue; + } + assert readState.version == version; + highestShardState = readState; highestShardVersion = version; - highestShardFile = stateFile; } } catch (Exception e) { logger.debug("[{}][{}]: failed to read [" + stateFile.getAbsolutePath() + "], ignoring...", e, shardId.index().name(), shardId.id()); @@ -202,11 +206,11 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste } } // did we find a state file? - if (highestShardFile == null) { + if (highestShardState == null) { continue; } - shardsState.put(shardId, new ShardStateInfo(highestShardVersion)); + shardsState.put(shardId, highestShardState); // update the global version if (highestShardVersion > highestVersion) { @@ -219,15 +223,17 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste } } - private long readShardState(byte[] data) throws Exception { + @Nullable + private ShardStateInfo readShardState(byte[] data) throws Exception { XContentParser parser = null; try { parser = XContentHelper.createParser(data, 0, data.length); XContentParser.Token token = parser.nextToken(); if (token == null) { - return -1; + return null; } long version = -1; + Boolean primary = null; String currentFieldName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -235,10 +241,12 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste } else if (token.isValue()) { if ("version".equals(currentFieldName)) { version = parser.longValue(); + } else if ("primary".equals(currentFieldName)) { + primary = parser.booleanValue(); } } } - return version; + return new ShardStateInfo(version, primary); } finally { if (parser != null) { parser.close(); @@ -425,7 +433,7 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste } } } - shardsState.put(new ShardId(shardIndex, shardId), new ShardStateInfo(version)); + shardsState.put(new ShardId(shardIndex, shardId), new ShardStateInfo(version, null)); } } } diff --git a/src/main/java/org/elasticsearch/gateway/local/state/shards/ShardStateInfo.java b/src/main/java/org/elasticsearch/gateway/local/state/shards/ShardStateInfo.java index 764ba936ad2..47426aa1e93 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/shards/ShardStateInfo.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/shards/ShardStateInfo.java @@ -19,13 +19,20 @@ package org.elasticsearch.gateway.local.state.shards; +import org.elasticsearch.common.Nullable; + /** */ public class ShardStateInfo { public final long version; - public ShardStateInfo(long version) { + // can be null if we don't know... + @Nullable + public final Boolean primary; + + public ShardStateInfo(long version, Boolean primary) { this.version = version; + this.primary = primary; } } From 7625f647ae19bb9494161b824e3490cc9c331a89 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 23 Jan 2012 21:58:28 +0200 Subject: [PATCH 196/270] also store the fact that a shard is primary or not in the shard state --- .../gateway/local/state/shards/LocalGatewayShardsState.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java b/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java index c7ad004b93b..81ffef6a1f6 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java @@ -262,6 +262,9 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste builder.prettyPrint(); builder.startObject(); builder.field("version", shardStateInfo.version); + if (shardStateInfo.primary != null) { + builder.field("primary", shardStateInfo.primary); + } builder.endObject(); builder.flush(); From 30b8823e3dad231ade5a49d3717200fcf041ddd1 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 23 Jan 2012 22:50:03 +0200 Subject: [PATCH 197/270] allow for plugins to set shards allocator --- .../allocation/allocator/ShardsAllocatorModule.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocatorModule.java b/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocatorModule.java index ad40076cfeb..71c3860de48 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocatorModule.java +++ b/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocatorModule.java @@ -29,6 +29,8 @@ public class ShardsAllocatorModule extends AbstractModule { private Settings settings; + private Class shardsAllocator; + private Class gatewayAllocator = NoneGatewayAllocator.class; public ShardsAllocatorModule(Settings settings) { @@ -39,9 +41,13 @@ public class ShardsAllocatorModule extends AbstractModule { this.gatewayAllocator = gatewayAllocator; } + public void setShardsAllocator(Class shardsAllocator) { + this.shardsAllocator = shardsAllocator; + } + @Override protected void configure() { bind(GatewayAllocator.class).to(gatewayAllocator).asEagerSingleton(); - bind(ShardsAllocator.class).to(EvenShardsCountAllocator.class).asEagerSingleton(); + bind(ShardsAllocator.class).to(shardsAllocator == null ? EvenShardsCountAllocator.class : shardsAllocator).asEagerSingleton(); } } From 7bcee7660a0c43f4692b6965db3bd5ab95cf5868 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 24 Jan 2012 12:56:47 +0200 Subject: [PATCH 198/270] better failure message when no master found (how long we waited for it) --- .../support/master/TransportMasterNodeOperationAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeOperationAction.java b/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeOperationAction.java index a058787c256..27be335e2c8 100644 --- a/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeOperationAction.java @@ -166,7 +166,7 @@ public abstract class TransportMasterNodeOperationAction Date: Tue, 24 Jan 2012 13:07:58 +0200 Subject: [PATCH 199/270] better failure message when no master found (how long we waited for it) --- .../elasticsearch/discovery/MasterNotDiscoveredException.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/elasticsearch/discovery/MasterNotDiscoveredException.java b/src/main/java/org/elasticsearch/discovery/MasterNotDiscoveredException.java index 218cecf83ee..b185ba50fc8 100644 --- a/src/main/java/org/elasticsearch/discovery/MasterNotDiscoveredException.java +++ b/src/main/java/org/elasticsearch/discovery/MasterNotDiscoveredException.java @@ -31,6 +31,10 @@ public class MasterNotDiscoveredException extends ElasticSearchException { super(""); } + public MasterNotDiscoveredException(String message) { + super(message); + } + @Override public RestStatus status() { return RestStatus.SERVICE_UNAVAILABLE; From e0f9f292ec02a096a829dc42dffc3bbfdd9d6c4b Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 24 Jan 2012 13:08:16 +0200 Subject: [PATCH 200/270] better logging (port number) when failing to bind --- .../discovery/zen/ping/multicast/MulticastZenPing.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java b/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java index ef5affae086..0b025f2c341 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java @@ -183,7 +183,7 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem multicastSocket.close(); multicastSocket = null; } - logger.warn("disabled, failed to setup multicast discovery on {}: {}", multicastInterface, e.getMessage()); + logger.warn("disabled, failed to setup multicast discovery on port [{}], [{}]: {}", port, multicastInterface, e.getMessage()); if (logger.isDebugEnabled()) { logger.debug("disabled, failed to setup multicast discovery on {}", e, multicastInterface); } From c1a2a5c9104ea0463f72def13eb44c32bb75786a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 24 Jan 2012 13:10:38 +0200 Subject: [PATCH 201/270] close the multicast socket in test --- .../discovery/zen/ping/multicast/MulticastZenPingTests.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/elasticsearch/test/unit/discovery/zen/ping/multicast/MulticastZenPingTests.java b/src/test/java/org/elasticsearch/test/unit/discovery/zen/ping/multicast/MulticastZenPingTests.java index 3b3c47e9276..fd361b36673 100644 --- a/src/test/java/org/elasticsearch/test/unit/discovery/zen/ping/multicast/MulticastZenPingTests.java +++ b/src/test/java/org/elasticsearch/test/unit/discovery/zen/ping/multicast/MulticastZenPingTests.java @@ -121,9 +121,10 @@ public class MulticastZenPingTests { }); zenPingA.start(); + MulticastSocket multicastSocket = null; try { Loggers.getLogger(MulticastZenPing.class).setLevel("TRACE"); - MulticastSocket multicastSocket = new MulticastSocket(54328); + multicastSocket = new MulticastSocket(54328); multicastSocket.setReceiveBufferSize(2048); multicastSocket.setSendBufferSize(2048); multicastSocket.setSoTimeout(60000); @@ -135,6 +136,7 @@ public class MulticastZenPingTests { Thread.sleep(100); } finally { Loggers.getLogger(MulticastZenPing.class).setLevel("INFO"); + if (multicastSocket != null) multicastSocket.close(); zenPingA.close(); threadPool.shutdown(); } From 68a0dca577f53a7b4701708e36b2ee4f1f071ff1 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 24 Jan 2012 13:28:15 +0200 Subject: [PATCH 202/270] add the version the index was created with (as a setting) to support better upgrade in the future between versions --- .../cluster/metadata/IndexMetaData.java | 1 + .../metadata/MetaDataCreateIndexService.java | 4 ++++ .../common/settings/ImmutableSettings.java | 19 +++++++++++++++++++ .../common/settings/Settings.java | 6 ++++++ .../state/meta/LocalGatewayMetaState.java | 7 ++++++- 5 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 3ed61adf271..05b096c1eca 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -115,6 +115,7 @@ public class IndexMetaData { public static final String SETTING_NUMBER_OF_REPLICAS = "index.number_of_replicas"; public static final String SETTING_AUTO_EXPAND_REPLICAS = "index.auto_expand_replicas"; public static final String SETTING_READ_ONLY = "index.blocks.read_only"; + public static final String SETTING_VERSION_CREATED = "index.version.created"; private final String index; private final long version; diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 5138cdaf0c2..2c36ae60ae0 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -24,6 +24,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ProcessedClusterStateUpdateTask; @@ -211,6 +212,9 @@ public class MetaDataCreateIndexService extends AbstractComponent { } } } + + indexSettingsBuilder.put(SETTING_VERSION_CREATED, Version.CURRENT); + Settings actualIndexSettings = indexSettingsBuilder.build(); // Set up everything, now locally create the index to see that things are ok, and apply diff --git a/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java b/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java index 70f605a52eb..2679e19683e 100644 --- a/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java +++ b/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java @@ -21,6 +21,7 @@ package org.elasticsearch.common.settings; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import org.elasticsearch.Version; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Classes; import org.elasticsearch.common.Strings; @@ -304,6 +305,19 @@ public class ImmutableSettings implements Settings { return Collections.unmodifiableMap(retVal); } + @Override + public Version getAsVersion(String setting, Version defaultVersion) throws SettingsException { + String sValue = get(setting); + if (sValue == null) { + return defaultVersion; + } + try { + return Version.fromId(Integer.parseInt(sValue)); + } catch (Exception e) { + throw new SettingsException("Failed to parse version setting [" + setting + "] with value [" + sValue + "]", e); + } + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -440,6 +454,11 @@ public class ImmutableSettings implements Settings { return this; } + public Builder put(String setting, Version version) { + put(setting, version.id); + return this; + } + /** * Sets the setting with the provided setting key and the long value. * diff --git a/src/main/java/org/elasticsearch/common/settings/Settings.java b/src/main/java/org/elasticsearch/common/settings/Settings.java index cc05aa17b78..159a727f1d1 100644 --- a/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -20,6 +20,7 @@ package org.elasticsearch.common.settings; import com.google.common.collect.ImmutableMap; +import org.elasticsearch.Version; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.SizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -231,6 +232,11 @@ public interface Settings { */ String[] getAsArray(String settingPrefix) throws SettingsException; + /** + * Retruns a parsed version. + */ + Version getAsVersion(String setting, Version defaultVersion) throws SettingsException; + /** * A settings builder interface. */ diff --git a/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java index dab24bd85d8..f94d914f6ec 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java @@ -20,6 +20,7 @@ package org.elasticsearch.gateway.local.state.meta; import com.google.common.io.Closeables; +import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -30,6 +31,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.CachedStreamOutput; +import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.*; @@ -421,7 +423,10 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS writeGlobalState("upgrade", MetaData.builder().metaData(metaData).version(version).build(), null); for (IndexMetaData indexMetaData : metaData) { - writeIndex("upgrade", IndexMetaData.newIndexMetaDataBuilder(indexMetaData).version(version).build(), null); + IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.newIndexMetaDataBuilder(indexMetaData).version(version); + // set the created version to 0.18 + indexMetaDataBuilder.settings(ImmutableSettings.settingsBuilder().put(indexMetaData.settings()).put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_18_0)); + writeIndex("upgrade", indexMetaDataBuilder.build(), null); } // rename shards state to backup state From 01d8305af3dc277753eaee7c9c2c6d417fc9e01b Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 24 Jan 2012 19:52:04 +0200 Subject: [PATCH 203/270] better failure message when failing to load settings --- .../org/elasticsearch/common/settings/ImmutableSettings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java b/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java index 2679e19683e..d1d89de1553 100644 --- a/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java +++ b/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java @@ -593,7 +593,7 @@ public class ImmutableSettings implements Settings { Map loadedSettings = settingsLoader.load(source); put(loadedSettings); } catch (Exception e) { - throw new SettingsException("Failed to load settings from [" + source + "]"); + throw new SettingsException("Failed to load settings from [" + source + "]", e); } return this; } From 68bb5d14346cc4b74fac0bfb92e6ede1162ad22b Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 25 Jan 2012 11:58:29 +0200 Subject: [PATCH 204/270] by default, index metadata to be stored in smile format and store binary format mapping and alias filter to improve the cost it takes to persist them --- .../cluster/metadata/AliasMetaData.java | 25 ++++++++--- .../cluster/metadata/IndexMetaData.java | 45 ++++++++++++++----- .../cluster/metadata/MappingMetaData.java | 25 +++++++++-- .../common/compress/CompressedString.java | 4 ++ .../state/meta/LocalGatewayMetaState.java | 23 +++++++--- .../SimpleRecoveryLocalGatewayTests.java | 4 ++ 6 files changed, 103 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java index 728e5a5d593..f0f890b6813 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java @@ -139,6 +139,11 @@ public class AliasMetaData { return alias; } + public Builder filter(CompressedString filter) { + this.filter = filter; + return this; + } + public Builder filter(String filter) { if (!Strings.hasLength(filter)) { this.filter = null; @@ -201,12 +206,18 @@ public class AliasMetaData { public static void toXContent(AliasMetaData aliasMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(aliasMetaData.alias(), XContentBuilder.FieldCaseConversion.NONE); + boolean binary = params.paramAsBoolean("binary", false); + if (aliasMetaData.filter() != null) { - byte[] data = aliasMetaData.filter().uncompressed(); - XContentParser parser = XContentFactory.xContent(data).createParser(data); - Map filter = parser.mapOrdered(); - parser.close(); - builder.field("filter", filter); + if (binary) { + builder.field("filter", aliasMetaData.filter.compressed()); + } else { + byte[] data = aliasMetaData.filter().uncompressed(); + XContentParser parser = XContentFactory.xContent(data).createParser(data); + Map filter = parser.mapOrdered(); + parser.close(); + builder.field("filter", filter); + } } if (aliasMetaData.indexRouting() != null) { builder.field("index_routing", aliasMetaData.indexRouting()); @@ -235,6 +246,10 @@ public class AliasMetaData { Map filter = parser.mapOrdered(); builder.filter(filter); } + } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + if ("filter".equals(currentFieldName)) { + builder.filter(new CompressedString(parser.binaryValue())); + } } else if (token == XContentParser.Token.VALUE_STRING) { if ("routing".equals(currentFieldName)) { builder.routing(parser.text()); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 05b096c1eca..bfb707a12be 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeFilters; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.regex.Regex; @@ -346,7 +347,7 @@ public class IndexMetaData { public Builder putMapping(String type, String source) throws IOException { XContentParser parser = XContentFactory.xContent(source).createParser(source); try { - putMapping(new MappingMetaData(type, parser.map())); + putMapping(new MappingMetaData(type, parser.mapOrdered())); } finally { parser.close(); } @@ -413,6 +414,8 @@ public class IndexMetaData { builder.field("version", indexMetaData.version()); builder.field("state", indexMetaData.state().toString().toLowerCase()); + boolean binary = params.paramAsBoolean("binary", false); + builder.startObject("settings"); for (Map.Entry entry : indexMetaData.settings().getAsMap().entrySet()) { builder.field(entry.getKey(), entry.getValue()); @@ -421,11 +424,15 @@ public class IndexMetaData { builder.startArray("mappings"); for (Map.Entry entry : indexMetaData.mappings().entrySet()) { - byte[] data = entry.getValue().source().uncompressed(); - XContentParser parser = XContentFactory.xContent(data).createParser(data); - Map mapping = parser.mapOrdered(); - parser.close(); - builder.map(mapping); + if (binary) { + builder.value(entry.getValue().source().compressed()); + } else { + byte[] data = entry.getValue().source().uncompressed(); + XContentParser parser = XContentFactory.xContent(data).createParser(data); + Map mapping = parser.mapOrdered(); + parser.close(); + builder.map(mapping); + } } builder.endArray(); @@ -462,10 +469,14 @@ public class IndexMetaData { builder.settings(settingsBuilder.build()); } else if ("mappings".equals(currentFieldName)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - Map mapping = parser.mapOrdered(); - if (mapping.size() == 1) { - String mappingType = mapping.keySet().iterator().next(); - builder.putMapping(new MappingMetaData(mappingType, mapping)); + if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + builder.putMapping(new MappingMetaData(new CompressedString(parser.binaryValue()))); + } else { + Map mapping = parser.mapOrdered(); + if (mapping.size() == 1) { + String mappingType = mapping.keySet().iterator().next(); + builder.putMapping(new MappingMetaData(mappingType, mapping)); + } } } } else if ("aliases".equals(currentFieldName)) { @@ -473,6 +484,20 @@ public class IndexMetaData { builder.putAlias(AliasMetaData.Builder.fromXContent(parser)); } } + } else if (token == XContentParser.Token.START_ARRAY) { + if ("mappings".equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + builder.putMapping(new MappingMetaData(new CompressedString(parser.binaryValue()))); + } else { + Map mapping = parser.mapOrdered(); + if (mapping.size() == 1) { + String mappingType = mapping.keySet().iterator().next(); + builder.putMapping(new MappingMetaData(mappingType, mapping)); + } + } + } + } } else if (token.isValue()) { if ("state".equals(currentFieldName)) { builder.state(State.fromString(parser.text())); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java index 7d6e38bf782..15f9aeeeeb4 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.ElasticSearchIllegalStateException; import org.elasticsearch.action.TimestampParsingException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; @@ -250,9 +251,9 @@ public class MappingMetaData { private final CompressedString source; - private final Id id; - private final Routing routing; - private final Timestamp timestamp; + private Id id; + private Routing routing; + private Timestamp timestamp; public MappingMetaData(DocumentMapper docMapper) { this.type = docMapper.type(); @@ -262,6 +263,20 @@ public class MappingMetaData { this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format()); } + public MappingMetaData(CompressedString mapping) throws IOException { + this.source = mapping; + Map mappingMap = XContentHelper.createParser(mapping.compressed(), 0, mapping.compressed().length).mapOrderedAndClose(); + if (mappingMap.size() != 1) { + throw new ElasticSearchIllegalStateException("Can't derive type from mapping, no root type: " + mapping.string()); + } + this.type = mappingMap.keySet().iterator().next(); + initMappers((Map) mappingMap.get(this.type)); + } + + public MappingMetaData(Map mapping) throws IOException { + this(mapping.keySet().iterator().next(), mapping); + } + public MappingMetaData(String type, Map mapping) throws IOException { this.type = type; this.source = new CompressedString(XContentFactory.jsonBuilder().map(mapping).string()); @@ -269,6 +284,10 @@ public class MappingMetaData { if (mapping.size() == 1 && mapping.containsKey(type)) { withoutType = (Map) mapping.get(type); } + initMappers(withoutType); + } + + private void initMappers(Map withoutType) { if (withoutType.containsKey("_id")) { String path = null; Map routingNode = (Map) withoutType.get("_id"); diff --git a/src/main/java/org/elasticsearch/common/compress/CompressedString.java b/src/main/java/org/elasticsearch/common/compress/CompressedString.java index c06dba82569..b6230ade2ba 100644 --- a/src/main/java/org/elasticsearch/common/compress/CompressedString.java +++ b/src/main/java/org/elasticsearch/common/compress/CompressedString.java @@ -40,6 +40,10 @@ public class CompressedString implements Streamable { CompressedString() { } + public CompressedString(byte[] compressed) { + this.bytes = compressed; + } + public CompressedString(String str) throws IOException { UnicodeUtil.UTF8Result result = Unicode.unsafeFromStringAsUtf8(str); this.bytes = LZFEncoder.encode(result.result, result.length); diff --git a/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java index f94d914f6ec..daf32fdf1d5 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java @@ -19,6 +19,7 @@ package org.elasticsearch.gateway.local.state.meta; +import com.google.common.collect.Maps; import com.google.common.io.Closeables; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterChangedEvent; @@ -42,6 +43,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.util.Map; import java.util.Set; /** @@ -53,13 +55,24 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS private volatile MetaData currentMetaData; + private final XContentType format; + private final ToXContent.Params formatParams; + @Inject public LocalGatewayMetaState(Settings settings, NodeEnvironment nodeEnv, TransportNodesListGatewayMetaState nodesListGatewayMetaState) throws Exception { super(settings); this.nodeEnv = nodeEnv; - + this.format = XContentType.fromRestContentType(settings.get("format", "smile")); nodesListGatewayMetaState.init(this); + if (this.format == XContentType.SMILE) { + Map params = Maps.newHashMap(); + params.put("binary", "true"); + formatParams = new ToXContent.MapParams(params); + } else { + formatParams = ToXContent.EMPTY_PARAMS; + } + try { pre019Upgrade(); long start = System.currentTimeMillis(); @@ -150,9 +163,9 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS logger.trace("[{}] writing state, reason [{}]", indexMetaData.index(), reason); CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry(); try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, cachedEntry.cachedBytes()); + XContentBuilder builder = XContentFactory.contentBuilder(format, cachedEntry.cachedBytes()); builder.startObject(); - IndexMetaData.Builder.toXContent(indexMetaData, builder, ToXContent.EMPTY_PARAMS); + IndexMetaData.Builder.toXContent(indexMetaData, builder, formatParams); builder.endObject(); builder.flush(); @@ -201,9 +214,9 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry(); try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, cachedEntry.cachedBytes()); + XContentBuilder builder = XContentFactory.contentBuilder(format, cachedEntry.cachedBytes()); builder.startObject(); - MetaData.Builder.toXContent(globalMetaData, builder, ToXContent.EMPTY_PARAMS); + MetaData.Builder.toXContent(globalMetaData, builder, formatParams); builder.endObject(); builder.flush(); diff --git a/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java b/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java index 343cabfbab3..d4b46b051bc 100644 --- a/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java +++ b/src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.gateway.Gateway; +import org.elasticsearch.index.query.FilterBuilders; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.node.Node; import org.elasticsearch.node.internal.InternalNode; @@ -295,6 +296,7 @@ public class SimpleRecoveryLocalGatewayTests extends AbstractNodesTests { .startObject("field2").field("type", "string").field("store", "yes").field("index", "not_analyzed").endObject() .endObject().endObject().endObject()) .execute().actionGet(); + node2.client().admin().indices().prepareAliases().addAlias("test", "test_alias", FilterBuilders.termFilter("field", "value")).execute().actionGet(); logger.info("--> closing the second node"); @@ -318,6 +320,8 @@ public class SimpleRecoveryLocalGatewayTests extends AbstractNodesTests { ClusterState state = node1.client().admin().cluster().prepareState().execute().actionGet().state(); assertThat(state.metaData().index("test").mapping("type2"), notNullValue()); assertThat(state.metaData().templates().get("template_1").template(), equalTo("te*")); + assertThat(state.metaData().index("test").aliases().get("test_alias"), notNullValue()); + assertThat(state.metaData().index("test").aliases().get("test_alias").filter(), notNullValue()); } @Test From 7d757a2c2ed616527297f396d1a6ff385481afc9 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 25 Jan 2012 13:03:18 +0200 Subject: [PATCH 205/270] current search context is only needed in case of script --- .../index/query/CustomFiltersScoreQueryParser.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/query/CustomFiltersScoreQueryParser.java b/src/main/java/org/elasticsearch/index/query/CustomFiltersScoreQueryParser.java index 2d0d0249a35..a2bf912d31f 100644 --- a/src/main/java/org/elasticsearch/index/query/CustomFiltersScoreQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/CustomFiltersScoreQueryParser.java @@ -141,15 +141,15 @@ public class CustomFiltersScoreQueryParser implements QueryParser { throw new QueryParsingException(parseContext.index(), "[custom_filters_score] requires 'filters' field"); } - SearchContext context = SearchContext.current(); - if (context == null) { - throw new ElasticSearchIllegalStateException("No search context on going..."); - } FiltersFunctionScoreQuery.FilterFunction[] filterFunctions = new FiltersFunctionScoreQuery.FilterFunction[filters.size()]; for (int i = 0; i < filterFunctions.length; i++) { ScoreFunction scoreFunction; String script = scripts.get(i); if (script != null) { + SearchContext context = SearchContext.current(); + if (context == null) { + throw new ElasticSearchIllegalStateException("No search context on going..."); + } SearchScript searchScript = context.scriptService().search(context.lookup(), scriptLang, script, vars); scoreFunction = new CustomScoreQueryParser.ScriptScoreFunction(script, vars, searchScript); } else { From da433df217e2239fe874de00bd4b360f4427dc75 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 26 Jan 2012 00:18:46 +0200 Subject: [PATCH 206/270] Mapping: _source mapping to allow for format to convert to (if needed), closes #1639. --- .../common/xcontent/XContentType.java | 14 ++- .../xcontent/json/JsonXContentGenerator.java | 4 + .../mapper/internal/SourceFieldMapper.java | 113 +++++++++++++++--- .../source/DefaultSourceMappingTests.java | 74 +++++++++++- 4 files changed, 184 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index e656c74474e..173e8a335bc 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -21,8 +21,6 @@ package org.elasticsearch.common.xcontent; /** * The content type of {@link org.elasticsearch.common.xcontent.XContent}. - * - * */ public enum XContentType { @@ -34,6 +32,11 @@ public enum XContentType { public String restContentType() { return "application/json; charset=UTF-8"; } + + @Override + public String shortName() { + return "json"; + } }, /** * The jackson based smile binary format. Fast and compact binary format. @@ -43,6 +46,11 @@ public enum XContentType { public String restContentType() { return "application/smile"; } + + @Override + public String shortName() { + return "smile"; + } }; public static XContentType fromRestContentType(String contentType) { @@ -71,4 +79,6 @@ public enum XContentType { } public abstract String restContentType(); + + public abstract String shortName(); } diff --git a/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java index 2b7c6757aec..d5545241eb6 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java +++ b/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java @@ -267,6 +267,10 @@ public class JsonXContentGenerator implements XContentGenerator { @Override public void copyCurrentStructure(XContentParser parser) throws IOException { + // the start of the parser + if (parser.currentToken() == null) { + parser.nextToken(); + } if (parser instanceof JsonXContentParser) { generator.copyCurrentStructure(((JsonXContentParser) parser).parser); } else { diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java index 34ac7473b86..e6be08d1828 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper.internal; +import com.google.common.base.Objects; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Fieldable; @@ -27,9 +28,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.lzf.LZF; import org.elasticsearch.common.compress.lzf.LZFDecoder; -import org.elasticsearch.common.io.stream.CachedStreamOutput; -import org.elasticsearch.common.io.stream.LZFStreamOutput; -import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.*; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.document.ResetFieldSelector; import org.elasticsearch.common.unit.ByteSizeValue; @@ -46,6 +45,7 @@ import java.util.List; import java.util.Map; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; +import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringValue; import static org.elasticsearch.index.mapper.MapperBuilders.source; /** @@ -61,6 +61,7 @@ public class SourceFieldMapper extends AbstractFieldMapper implements In public static final String NAME = SourceFieldMapper.NAME; public static final boolean ENABLED = true; public static final long COMPRESS_THRESHOLD = -1; + public static final String FORMAT = null; // default format is to use the one provided public static final Field.Index INDEX = Field.Index.NO; public static final Field.Store STORE = Field.Store.YES; public static final boolean OMIT_NORMS = true; @@ -77,6 +78,8 @@ public class SourceFieldMapper extends AbstractFieldMapper implements In private Boolean compress = null; + private String format = Defaults.FORMAT; + private String[] includes = Defaults.INCLUDES; private String[] excludes = Defaults.EXCLUDES; @@ -99,6 +102,11 @@ public class SourceFieldMapper extends AbstractFieldMapper implements In return this; } + public Builder format(String format) { + this.format = format; + return this; + } + public Builder includes(String[] includes) { this.includes = includes; return this; @@ -111,7 +119,7 @@ public class SourceFieldMapper extends AbstractFieldMapper implements In @Override public SourceFieldMapper build(BuilderContext context) { - return new SourceFieldMapper(name, enabled, compress, compressThreshold, includes, excludes); + return new SourceFieldMapper(name, enabled, format, compress, compressThreshold, includes, excludes); } } @@ -135,6 +143,8 @@ public class SourceFieldMapper extends AbstractFieldMapper implements In builder.compressThreshold(ByteSizeValue.parseBytesSizeValue(fieldNode.toString()).bytes()); builder.compress(true); } + } else if ("format".equals(fieldName)) { + builder.format(nodeStringValue(fieldNode, null)); } else if (fieldName.equals("includes")) { List values = (List) fieldNode; String[] includes = new String[values.size()]; @@ -166,11 +176,15 @@ public class SourceFieldMapper extends AbstractFieldMapper implements In private String[] excludes; + private String format; + + private XContentType formatContentType; + public SourceFieldMapper() { - this(Defaults.NAME, Defaults.ENABLED, null, -1, Defaults.INCLUDES, Defaults.EXCLUDES); + this(Defaults.NAME, Defaults.ENABLED, Defaults.FORMAT, null, -1, Defaults.INCLUDES, Defaults.EXCLUDES); } - protected SourceFieldMapper(String name, boolean enabled, Boolean compress, long compressThreshold, String[] includes, String[] excludes) { + protected SourceFieldMapper(String name, boolean enabled, String format, Boolean compress, long compressThreshold, String[] includes, String[] excludes) { super(new Names(name, name, name, name), Defaults.INDEX, Defaults.STORE, Defaults.TERM_VECTOR, Defaults.BOOST, Defaults.OMIT_NORMS, Defaults.OMIT_TERM_FREQ_AND_POSITIONS, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER); this.enabled = enabled; @@ -178,6 +192,8 @@ public class SourceFieldMapper extends AbstractFieldMapper implements In this.compressThreshold = compressThreshold; this.includes = includes; this.excludes = excludes; + this.format = format; + this.formatContentType = format == null ? null : XContentType.fromRestContentType(format); } public boolean enabled() { @@ -239,7 +255,11 @@ public class SourceFieldMapper extends AbstractFieldMapper implements In } else { streamOutput = cachedEntry.cachedBytes(); } - XContentBuilder builder = XContentFactory.contentBuilder(mapTuple.v1(), streamOutput).map(filteredSource); + XContentType contentType = formatContentType; + if (contentType == null) { + contentType = mapTuple.v1(); + } + XContentBuilder builder = XContentFactory.contentBuilder(contentType, streamOutput).map(filteredSource); builder.close(); data = cachedEntry.bytes().copiedByteArray(); @@ -250,17 +270,71 @@ public class SourceFieldMapper extends AbstractFieldMapper implements In } else if (compress != null && compress && !LZF.isCompressed(data, dataOffset, dataLength)) { if (compressThreshold == -1 || dataLength > compressThreshold) { CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry(); - LZFStreamOutput streamOutput = cachedEntry.cachedLZFBytes(); - streamOutput.writeBytes(data, dataOffset, dataLength); - streamOutput.flush(); - // we copy over the byte array, since we need to push back the cached entry - // TODO, we we had a handle into when we are done with parsing, then we push back then and not copy over bytes - data = cachedEntry.bytes().copiedByteArray(); - dataOffset = 0; - dataLength = data.length; - CachedStreamOutput.pushEntry(cachedEntry); - // update the data in the context, so it can be compressed and stored compressed outside... - context.source(data, dataOffset, dataLength); + try { + XContentType contentType = XContentFactory.xContentType(data, dataOffset, dataLength); + if (formatContentType != null && formatContentType != contentType) { + XContentBuilder builder = XContentFactory.contentBuilder(formatContentType, cachedEntry.cachedLZFBytes()); + builder.copyCurrentStructure(XContentFactory.xContent(contentType).createParser(data, dataOffset, dataLength)); + builder.close(); + } else { + LZFStreamOutput streamOutput = cachedEntry.cachedLZFBytes(); + streamOutput.writeBytes(data, dataOffset, dataLength); + streamOutput.flush(); + } + // we copy over the byte array, since we need to push back the cached entry + // TODO, we we had a handle into when we are done with parsing, then we push back then and not copy over bytes + data = cachedEntry.bytes().copiedByteArray(); + dataOffset = 0; + dataLength = data.length; + // update the data in the context, so it can be compressed and stored compressed outside... + context.source(data, dataOffset, dataLength); + } finally { + CachedStreamOutput.pushEntry(cachedEntry); + } + } + } else if (formatContentType != null) { + // see if we need to convert the content type + if (LZF.isCompressed(data, dataOffset, dataLength)) { + BytesStreamInput siBytes = new BytesStreamInput(data, dataOffset, dataLength, false); + LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes); + XContentType contentType = XContentFactory.xContentType(siLzf); + siLzf.resetToBufferStart(); + if (contentType != formatContentType) { + // we need to reread and store back, compressed.... + CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry(); + try { + LZFStreamOutput streamOutput = cachedEntry.cachedLZFBytes(); + XContentBuilder builder = XContentFactory.contentBuilder(formatContentType, streamOutput); + builder.copyCurrentStructure(XContentFactory.xContent(contentType).createParser(siLzf)); + builder.close(); + data = cachedEntry.bytes().copiedByteArray(); + dataOffset = 0; + dataLength = data.length; + // update the data in the context, so we store it in the translog in this format + context.source(data, dataOffset, dataLength); + } finally { + CachedStreamOutput.pushEntry(cachedEntry); + } + } + } else { + XContentType contentType = XContentFactory.xContentType(data, dataOffset, dataLength); + if (contentType != formatContentType) { + // we need to reread and store back + // we need to reread and store back, compressed.... + CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry(); + try { + XContentBuilder builder = XContentFactory.contentBuilder(formatContentType, cachedEntry.cachedBytes()); + builder.copyCurrentStructure(XContentFactory.xContent(contentType).createParser(data, dataOffset, dataLength)); + builder.close(); + data = cachedEntry.bytes().copiedByteArray(); + dataOffset = 0; + dataLength = data.length; + // update the data in the context, so we store it in the translog in this format + context.source(data, dataOffset, dataLength); + } finally { + CachedStreamOutput.pushEntry(cachedEntry); + } + } } } return new Field(names().indexName(), data, dataOffset, dataLength); @@ -321,6 +395,9 @@ public class SourceFieldMapper extends AbstractFieldMapper implements In if (enabled != Defaults.ENABLED) { builder.field("enabled", enabled); } + if (!Objects.equal(format, Defaults.FORMAT)) { + builder.field("format", format); + } if (compress != null) { builder.field("compress", compress); } diff --git a/src/test/java/org/elasticsearch/test/unit/index/mapper/source/DefaultSourceMappingTests.java b/src/test/java/org/elasticsearch/test/unit/index/mapper/source/DefaultSourceMappingTests.java index 364b79946d9..476e9e7b5f6 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/mapper/source/DefaultSourceMappingTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/mapper/source/DefaultSourceMappingTests.java @@ -20,9 +20,14 @@ package org.elasticsearch.test.unit.index.mapper.source; import org.apache.lucene.document.Fieldable; +import org.elasticsearch.common.compress.lzf.LZF; +import org.elasticsearch.common.compress.lzf.LZFDecoder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.test.unit.index.mapper.MapperTests; import org.testng.annotations.Test; @@ -36,6 +41,73 @@ import static org.hamcrest.Matchers.equalTo; */ public class DefaultSourceMappingTests { + @Test + public void testNoFormat() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_source").endObject() + .endObject().endObject().string(); + + DocumentMapper documentMapper = MapperTests.newParser().parse(mapping); + ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject() + .field("field", "value") + .endObject().copiedBytes()); + + assertThat(XContentFactory.xContentType(doc.source(), doc.sourceOffset(), doc.sourceLength()), equalTo(XContentType.JSON)); + + documentMapper = MapperTests.newParser().parse(mapping); + doc = documentMapper.parse("type", "1", XContentFactory.smileBuilder().startObject() + .field("field", "value") + .endObject().copiedBytes()); + + assertThat(XContentFactory.xContentType(doc.source(), doc.sourceOffset(), doc.sourceLength()), equalTo(XContentType.SMILE)); + } + + @Test + public void testJsonFormat() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_source").field("format", "json").endObject() + .endObject().endObject().string(); + + DocumentMapper documentMapper = MapperTests.newParser().parse(mapping); + ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject() + .field("field", "value") + .endObject().copiedBytes()); + + assertThat(XContentFactory.xContentType(doc.source(), doc.sourceOffset(), doc.sourceLength()), equalTo(XContentType.JSON)); + + documentMapper = MapperTests.newParser().parse(mapping); + doc = documentMapper.parse("type", "1", XContentFactory.smileBuilder().startObject() + .field("field", "value") + .endObject().copiedBytes()); + + assertThat(XContentFactory.xContentType(doc.source(), doc.sourceOffset(), doc.sourceLength()), equalTo(XContentType.JSON)); + } + + @Test + public void testJsonFormatCompressed() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_source").field("format", "json").field("compress", true).endObject() + .endObject().endObject().string(); + + DocumentMapper documentMapper = MapperTests.newParser().parse(mapping); + ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject() + .field("field", "value") + .endObject().copiedBytes()); + + assertThat(LZF.isCompressed(doc.source(), doc.sourceOffset(), doc.sourceLength()), equalTo(true)); + byte[] uncompressed = LZFDecoder.decode(doc.source(), doc.sourceOffset(), doc.sourceLength()); + assertThat(XContentFactory.xContentType(uncompressed), equalTo(XContentType.JSON)); + + documentMapper = MapperTests.newParser().parse(mapping); + doc = documentMapper.parse("type", "1", XContentFactory.smileBuilder().startObject() + .field("field", "value") + .endObject().copiedBytes()); + + assertThat(LZF.isCompressed(doc.source(), doc.sourceOffset(), doc.sourceLength()), equalTo(true)); + uncompressed = LZFDecoder.decode(doc.source(), doc.sourceOffset(), doc.sourceLength()); + assertThat(XContentFactory.xContentType(uncompressed), equalTo(XContentType.JSON)); + } + @Test public void testIncludeExclude() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") From 78722a51f71617e189b7035e021eafca917b6370 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 26 Jan 2012 13:52:43 +0200 Subject: [PATCH 207/270] add a comment option to another jvm flag that only applies in 7 --- bin/elasticsearch.bat | 3 +++ bin/elasticsearch.in.sh | 3 +++ 2 files changed, 6 insertions(+) diff --git a/bin/elasticsearch.bat b/bin/elasticsearch.bat index 90e45e23621..00388e7d0aa 100644 --- a/bin/elasticsearch.bat +++ b/bin/elasticsearch.bat @@ -33,6 +33,9 @@ set JAVA_OPTS=%JAVA_OPTS% -XX:MaxTenuringThreshold=1 set JAVA_OPTS=%JAVA_OPTS% -XX:CMSInitiatingOccupancyFraction=75 set JAVA_OPTS=%JAVA_OPTS% -XX:+UseCMSInitiatingOccupancyOnly +REM When running under Java 7 +REM JAVA_OPTS=%JAVA_OPTS% -XX:+UseCondCardMark + REM GC logging options -- uncomment to enable REM JAVA_OPTS=%JAVA_OPTS% -XX:+PrintGCDetails REM JAVA_OPTS=%JAVA_OPTS% -XX:+PrintGCTimeStamps diff --git a/bin/elasticsearch.in.sh b/bin/elasticsearch.in.sh index b3230b58184..7ee2c9b039f 100644 --- a/bin/elasticsearch.in.sh +++ b/bin/elasticsearch.in.sh @@ -32,6 +32,9 @@ JAVA_OPTS="$JAVA_OPTS -XX:MaxTenuringThreshold=1" JAVA_OPTS="$JAVA_OPTS -XX:CMSInitiatingOccupancyFraction=75" JAVA_OPTS="$JAVA_OPTS -XX:+UseCMSInitiatingOccupancyOnly" +# When running under Java 7 +#JAVA_OPTS="$JAVA_OPTS -XX:+UseCondCardMark" + # GC logging options -- uncomment to enable # JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDetails" # JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCTimeStamps" From 293a971ea6a6b4d708b812188d673945ecfaaf50 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 26 Jan 2012 13:53:37 +0200 Subject: [PATCH 208/270] Better failure when passing invalid options to a query/filter, closes #1640. --- .../index/query/AndFilterParser.java | 4 +++- .../index/query/BoolFilterParser.java | 6 +++++ .../index/query/BoolQueryParser.java | 6 +++++ .../index/query/BoostingQueryParser.java | 4 ++++ .../index/query/ConstantScoreQueryParser.java | 4 ++++ .../query/CustomBoostFactorQueryParser.java | 4 ++++ .../query/CustomFiltersScoreQueryParser.java | 8 ++++++- .../index/query/CustomScoreQueryParser.java | 4 ++++ .../index/query/DisMaxQueryParser.java | 6 +++++ .../index/query/ExistsFilterParser.java | 2 ++ .../index/query/FQueryFilterParser.java | 6 +++-- .../query/FieldMaskingSpanQueryParser.java | 6 ++++- .../index/query/FieldQueryParser.java | 6 ++++- .../index/query/FilteredQueryParser.java | 4 ++++ .../query/FuzzyLikeThisFieldQueryParser.java | 15 ++++++++---- .../index/query/FuzzyLikeThisQueryParser.java | 6 +++-- .../index/query/FuzzyQueryParser.java | 6 ++++- .../query/GeoBoundingBoxFilterParser.java | 2 ++ .../index/query/GeoDistanceFilterParser.java | 4 ++-- .../index/query/GeoPolygonFilterParser.java | 6 +++-- .../index/query/HasChildFilterParser.java | 15 ++++++++---- .../index/query/HasChildQueryParser.java | 15 ++++++++---- .../index/query/IdsFilterParser.java | 4 ++++ .../index/query/IdsQueryParser.java | 4 ++++ .../index/query/IndicesQueryParser.java | 6 +++++ .../index/query/LimitFilterParser.java | 2 ++ .../index/query/MatchAllQueryParser.java | 2 ++ .../index/query/MissingFilterParser.java | 2 ++ .../query/MoreLikeThisFieldQueryParser.java | 4 ++++ .../index/query/MoreLikeThisQueryParser.java | 4 ++++ .../index/query/NestedFilterParser.java | 4 ++++ .../index/query/NestedQueryParser.java | 4 ++++ .../index/query/NotFilterParser.java | 2 ++ .../index/query/NumericRangeFilterParser.java | 4 ++++ .../index/query/OrFilterParser.java | 2 ++ .../index/query/PrefixQueryParser.java | 6 ++++- .../index/query/QueryParseContext.java | 24 ++++++++++++++----- .../index/query/QueryStringQueryParser.java | 4 ++++ .../index/query/RangeFilterParser.java | 4 ++++ .../index/query/RangeQueryParser.java | 14 +++++++++-- .../index/query/ScriptFilterParser.java | 4 ++++ .../index/query/SpanFirstQueryParser.java | 4 ++++ .../index/query/SpanNearQueryParser.java | 6 +++++ .../index/query/SpanNotQueryParser.java | 4 ++++ .../index/query/SpanOrQueryParser.java | 4 ++++ .../index/query/SpanTermQueryParser.java | 2 ++ .../index/query/TermQueryParser.java | 6 ++++- .../index/query/TermsFilterParser.java | 2 ++ .../index/query/TermsQueryParser.java | 4 ++-- .../index/query/TextQueryParser.java | 6 ++++- .../index/query/TopChildrenQueryParser.java | 4 ++++ .../index/query/TypeFilterParser.java | 6 ++--- .../index/query/WildcardQueryParser.java | 6 ++++- .../index/query/WrapperQueryParser.java | 8 +++++-- 54 files changed, 257 insertions(+), 44 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/query/AndFilterParser.java b/src/main/java/org/elasticsearch/index/query/AndFilterParser.java index 2b58832da14..d17786519b0 100644 --- a/src/main/java/org/elasticsearch/index/query/AndFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/AndFilterParser.java @@ -83,13 +83,15 @@ public class AndFilterParser implements FilterParser { filterName = parser.text(); } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { cacheKey = new CacheKeyFilter.Key(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[and] filter does not support [" + currentFieldName + "]"); } } } } if (filters.isEmpty()) { - throw new QueryParsingException(parseContext.index(), "[or] filter requires 'filters' to be set on it'"); + throw new QueryParsingException(parseContext.index(), "[and] filter requires 'filters' to be set on it'"); } // no need to cache this one diff --git a/src/main/java/org/elasticsearch/index/query/BoolFilterParser.java b/src/main/java/org/elasticsearch/index/query/BoolFilterParser.java index 5aa489ae4dc..f38244a1ca3 100644 --- a/src/main/java/org/elasticsearch/index/query/BoolFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/BoolFilterParser.java @@ -67,6 +67,8 @@ public class BoolFilterParser implements FilterParser { boolFilter.add(new FilterClause(parseContext.parseInnerFilter(), BooleanClause.Occur.MUST_NOT)); } else if ("should".equals(currentFieldName)) { boolFilter.add(new FilterClause(parseContext.parseInnerFilter(), BooleanClause.Occur.SHOULD)); + } else { + throw new QueryParsingException(parseContext.index(), "[bool] filter does not support [" + currentFieldName + "]"); } } else if (token == XContentParser.Token.START_ARRAY) { if ("must".equals(currentFieldName)) { @@ -81,6 +83,8 @@ public class BoolFilterParser implements FilterParser { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { boolFilter.add(new FilterClause(parseContext.parseInnerFilter(), BooleanClause.Occur.SHOULD)); } + } else { + throw new QueryParsingException(parseContext.index(), "[bool] filter does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("_cache".equals(currentFieldName)) { @@ -89,6 +93,8 @@ public class BoolFilterParser implements FilterParser { filterName = parser.text(); } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { cacheKey = new CacheKeyFilter.Key(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[bool] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java b/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java index 7667c3ee13c..67948b5db67 100644 --- a/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java @@ -72,6 +72,8 @@ public class BoolQueryParser implements QueryParser { clauses.add(new BooleanClause(parseContext.parseInnerQuery(), BooleanClause.Occur.MUST_NOT)); } else if ("should".equals(currentFieldName)) { clauses.add(new BooleanClause(parseContext.parseInnerQuery(), BooleanClause.Occur.SHOULD)); + } else { + throw new QueryParsingException(parseContext.index(), "[bool] query does not support [" + currentFieldName + "]"); } } else if (token == XContentParser.Token.START_ARRAY) { if ("must".equals(currentFieldName)) { @@ -86,6 +88,8 @@ public class BoolQueryParser implements QueryParser { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { clauses.add(new BooleanClause(parseContext.parseInnerQuery(), BooleanClause.Occur.SHOULD)); } + } else { + throw new QueryParsingException(parseContext.index(), "bool query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("disable_coord".equals(currentFieldName) || "disableCoord".equals(currentFieldName)) { @@ -96,6 +100,8 @@ public class BoolQueryParser implements QueryParser { minimumNumberShouldMatch = parser.intValue(); } else if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[bool] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/BoostingQueryParser.java b/src/main/java/org/elasticsearch/index/query/BoostingQueryParser.java index 1c49c0947cb..091e4ac4cfe 100644 --- a/src/main/java/org/elasticsearch/index/query/BoostingQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/BoostingQueryParser.java @@ -61,12 +61,16 @@ public class BoostingQueryParser implements QueryParser { positiveQuery = parseContext.parseInnerQuery(); } else if ("negative".equals(currentFieldName)) { negativeQuery = parseContext.parseInnerQuery(); + } else { + throw new QueryParsingException(parseContext.index(), "[boosting] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("negative_boost".equals(currentFieldName) || "negativeBoost".equals(currentFieldName)) { negativeBoost = parser.floatValue(); } else if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[boosting] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryParser.java b/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryParser.java index 03114031e1c..a39d1b5afe5 100644 --- a/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryParser.java @@ -66,6 +66,8 @@ public class ConstantScoreQueryParser implements QueryParser { filter = parseContext.parseInnerFilter(); } else if ("query".equals(currentFieldName)) { query = parseContext.parseInnerQuery(); + } else { + throw new QueryParsingException(parseContext.index(), "[constant_score] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("boost".equals(currentFieldName)) { @@ -74,6 +76,8 @@ public class ConstantScoreQueryParser implements QueryParser { cache = parser.booleanValue(); } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { cacheKey = new CacheKeyFilter.Key(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[constant_score] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/CustomBoostFactorQueryParser.java b/src/main/java/org/elasticsearch/index/query/CustomBoostFactorQueryParser.java index 863718b24bc..ba9e25fc3ef 100644 --- a/src/main/java/org/elasticsearch/index/query/CustomBoostFactorQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/CustomBoostFactorQueryParser.java @@ -60,12 +60,16 @@ public class CustomBoostFactorQueryParser implements QueryParser { } else if (token == XContentParser.Token.START_OBJECT) { if ("query".equals(currentFieldName)) { query = parseContext.parseInnerQuery(); + } else { + throw new QueryParsingException(parseContext.index(), "[custom_boost_factor] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("boost_factor".equals(currentFieldName) || "boostFactor".equals(currentFieldName)) { boostFactor = parser.floatValue(); } else if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[custom_boost_factor] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/CustomFiltersScoreQueryParser.java b/src/main/java/org/elasticsearch/index/query/CustomFiltersScoreQueryParser.java index a2bf912d31f..8d4c36ce668 100644 --- a/src/main/java/org/elasticsearch/index/query/CustomFiltersScoreQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/CustomFiltersScoreQueryParser.java @@ -76,6 +76,8 @@ public class CustomFiltersScoreQueryParser implements QueryParser { query = parseContext.parseInnerQuery(); } else if ("params".equals(currentFieldName)) { vars = parser.map(); + } else { + throw new QueryParsingException(parseContext.index(), "[custom_filters_score] query does not support [" + currentFieldName + "]"); } } else if (token == XContentParser.Token.START_ARRAY) { if ("filters".equals(currentFieldName)) { @@ -108,6 +110,8 @@ public class CustomFiltersScoreQueryParser implements QueryParser { scripts.add(script); boosts.add(fboost); } + } else { + throw new QueryParsingException(parseContext.index(), "[custom_filters_score] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("lang".equals(currentFieldName)) { @@ -129,8 +133,10 @@ public class CustomFiltersScoreQueryParser implements QueryParser { } else if ("first".equals(sScoreMode)) { scoreMode = FiltersFunctionScoreQuery.ScoreMode.First; } else { - throw new QueryParsingException(parseContext.index(), "illegal score_mode for nested query [" + sScoreMode + "]"); + throw new QueryParsingException(parseContext.index(), "[custom_filters_score] illegal score_mode [" + sScoreMode + "]"); } + } else { + throw new QueryParsingException(parseContext.index(), "[custom_filters_score] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/CustomScoreQueryParser.java b/src/main/java/org/elasticsearch/index/query/CustomScoreQueryParser.java index e866f9e8615..253d8770e5f 100644 --- a/src/main/java/org/elasticsearch/index/query/CustomScoreQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/CustomScoreQueryParser.java @@ -70,6 +70,8 @@ public class CustomScoreQueryParser implements QueryParser { query = parseContext.parseInnerQuery(); } else if ("params".equals(currentFieldName)) { vars = parser.map(); + } else { + throw new QueryParsingException(parseContext.index(), "[custom_score] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("script".equals(currentFieldName)) { @@ -78,6 +80,8 @@ public class CustomScoreQueryParser implements QueryParser { scriptLang = parser.text(); } else if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[custom_score] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/DisMaxQueryParser.java b/src/main/java/org/elasticsearch/index/query/DisMaxQueryParser.java index 8618dfd3d22..b1fa2b0bcde 100644 --- a/src/main/java/org/elasticsearch/index/query/DisMaxQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/DisMaxQueryParser.java @@ -63,6 +63,8 @@ public class DisMaxQueryParser implements QueryParser { } else if (token == XContentParser.Token.START_OBJECT) { if ("queries".equals(currentFieldName)) { queries.add(parseContext.parseInnerQuery()); + } else { + throw new QueryParsingException(parseContext.index(), "[dis_max] query does not support [" + currentFieldName + "]"); } } else if (token == XContentParser.Token.START_ARRAY) { if ("queries".equals(currentFieldName)) { @@ -70,12 +72,16 @@ public class DisMaxQueryParser implements QueryParser { queries.add(parseContext.parseInnerQuery()); token = parser.nextToken(); } + } else { + throw new QueryParsingException(parseContext.index(), "[dis_max] query does not support [" + currentFieldName + "]"); } } else { if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); } else if ("tie_breaker".equals(currentFieldName) || "tieBreaker".equals(currentFieldName)) { tieBreaker = parser.floatValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[dis_max] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/ExistsFilterParser.java b/src/main/java/org/elasticsearch/index/query/ExistsFilterParser.java index e8613ae2f93..c6ef6825d5e 100644 --- a/src/main/java/org/elasticsearch/index/query/ExistsFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/ExistsFilterParser.java @@ -62,6 +62,8 @@ public class ExistsFilterParser implements FilterParser { fieldName = parser.text(); } else if ("_name".equals(currentFieldName)) { filterName = parser.text(); + } else { + throw new QueryParsingException(parseContext.index(), "[exists] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/FQueryFilterParser.java b/src/main/java/org/elasticsearch/index/query/FQueryFilterParser.java index 27499c957c1..bea20aa2bef 100644 --- a/src/main/java/org/elasticsearch/index/query/FQueryFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/FQueryFilterParser.java @@ -31,8 +31,6 @@ import java.io.IOException; /** * The "fquery" filter is the same as the {@link QueryFilterParser} except that it allows also to * associate a name with the query filter. - * - * */ public class FQueryFilterParser implements FilterParser { @@ -64,6 +62,8 @@ public class FQueryFilterParser implements FilterParser { } else if (token == XContentParser.Token.START_OBJECT) { if ("query".equals(currentFieldName)) { query = parseContext.parseInnerQuery(); + } else { + throw new QueryParsingException(parseContext.index(), "[fquery] filter does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("_name".equals(currentFieldName)) { @@ -72,6 +72,8 @@ public class FQueryFilterParser implements FilterParser { cache = parser.booleanValue(); } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { cacheKey = new CacheKeyFilter.Key(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[fquery] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryParser.java b/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryParser.java index 624f03ecc46..ff7e36779d6 100644 --- a/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryParser.java @@ -63,15 +63,19 @@ public class FieldMaskingSpanQueryParser implements QueryParser { if ("query".equals(currentFieldName)) { Query query = parseContext.parseInnerQuery(); if (!(query instanceof SpanQuery)) { - throw new QueryParsingException(parseContext.index(), "field_masking_span [query] must be of type span query"); + throw new QueryParsingException(parseContext.index(), "[field_masking_span] query] must be of type span query"); } inner = (SpanQuery) query; + } else { + throw new QueryParsingException(parseContext.index(), "[field_masking_span] query does not support [" + currentFieldName + "]"); } } else { if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); } else if ("field".equals(currentFieldName)) { field = parser.text(); + } else { + throw new QueryParsingException(parseContext.index(), "[field_masking_span] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java b/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java index a9e2e1606e3..c331e1e6386 100644 --- a/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java @@ -61,7 +61,9 @@ public class FieldQueryParser implements QueryParser { XContentParser parser = parseContext.parser(); XContentParser.Token token = parser.nextToken(); - assert token == XContentParser.Token.FIELD_NAME; + if (token != XContentParser.Token.FIELD_NAME) { + throw new QueryParsingException(parseContext.index(), "[field] query malformed, no field"); + } String fieldName = parser.currentName(); QueryParserSettings qpSettings = new QueryParserSettings(); @@ -113,6 +115,8 @@ public class FieldQueryParser implements QueryParser { qpSettings.rewriteMethod(QueryParsers.parseRewriteMethod(parser.textOrNull())); } else if ("minimum_should_match".equals(currentFieldName) || "minimumShouldMatch".equals(currentFieldName)) { qpSettings.minimumShouldMatch(parser.textOrNull()); + } else { + throw new QueryParsingException(parseContext.index(), "[field] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/FilteredQueryParser.java b/src/main/java/org/elasticsearch/index/query/FilteredQueryParser.java index 808e083805e..080ff0d3a8a 100644 --- a/src/main/java/org/elasticsearch/index/query/FilteredQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/FilteredQueryParser.java @@ -66,6 +66,8 @@ public class FilteredQueryParser implements QueryParser { query = parseContext.parseInnerQuery(); } else if ("filter".equals(currentFieldName)) { filter = parseContext.parseInnerFilter(); + } else { + throw new QueryParsingException(parseContext.index(), "[filtered] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("boost".equals(currentFieldName)) { @@ -74,6 +76,8 @@ public class FilteredQueryParser implements QueryParser { cache = parser.booleanValue(); } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { cacheKey = new CacheKeyFilter.Key(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[filtered] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/FuzzyLikeThisFieldQueryParser.java b/src/main/java/org/elasticsearch/index/query/FuzzyLikeThisFieldQueryParser.java index db24e45faa7..a9343cd10db 100644 --- a/src/main/java/org/elasticsearch/index/query/FuzzyLikeThisFieldQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/FuzzyLikeThisFieldQueryParser.java @@ -42,8 +42,6 @@ import static org.elasticsearch.index.query.support.QueryParsers.wrapSmartNameQu * } * } * - * - * */ public class FuzzyLikeThisFieldQueryParser implements QueryParser { @@ -71,12 +69,16 @@ public class FuzzyLikeThisFieldQueryParser implements QueryParser { Analyzer analyzer = null; XContentParser.Token token = parser.nextToken(); - assert token == XContentParser.Token.FIELD_NAME; + if (token != XContentParser.Token.FIELD_NAME) { + throw new QueryParsingException(parseContext.index(), "[flt_field] query malformed, no field"); + } String fieldName = parser.currentName(); // now, we move after the field name, which starts the object token = parser.nextToken(); - assert token == XContentParser.Token.START_OBJECT; + if (token != XContentParser.Token.START_OBJECT) { + throw new QueryParsingException(parseContext.index(), "[flt_field] query malformed, no start_object"); + } String currentFieldName = null; @@ -98,6 +100,8 @@ public class FuzzyLikeThisFieldQueryParser implements QueryParser { prefixLength = parser.intValue(); } else if ("analyzer".equals(currentFieldName)) { analyzer = parseContext.analysisService().analyzer(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[flt_field] query does not support [" + currentFieldName + "]"); } } } @@ -126,6 +130,9 @@ public class FuzzyLikeThisFieldQueryParser implements QueryParser { // move to the next end object, to close the field name token = parser.nextToken(); + if (token != XContentParser.Token.END_OBJECT) { + throw new QueryParsingException(parseContext.index(), "[flt_field] query malformed, no end_object"); + } assert token == XContentParser.Token.END_OBJECT; return wrapSmartNameQuery(query, smartNameFieldMappers, parseContext); diff --git a/src/main/java/org/elasticsearch/index/query/FuzzyLikeThisQueryParser.java b/src/main/java/org/elasticsearch/index/query/FuzzyLikeThisQueryParser.java index 5cb0f9acb76..60c2c8898c3 100644 --- a/src/main/java/org/elasticsearch/index/query/FuzzyLikeThisQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/FuzzyLikeThisQueryParser.java @@ -41,8 +41,6 @@ import java.util.List; * } * } * - * - * */ public class FuzzyLikeThisQueryParser implements QueryParser { @@ -90,6 +88,8 @@ public class FuzzyLikeThisQueryParser implements QueryParser { prefixLength = parser.intValue(); } else if ("analyzer".equals(currentFieldName)) { analyzer = parseContext.analysisService().analyzer(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[flt] query does not support [" + currentFieldName + "]"); } } else if (token == XContentParser.Token.START_ARRAY) { if ("fields".equals(currentFieldName)) { @@ -97,6 +97,8 @@ public class FuzzyLikeThisQueryParser implements QueryParser { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { fields.add(parseContext.indexName(parser.text())); } + } else { + throw new QueryParsingException(parseContext.index(), "[flt] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java b/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java index 1f57c03319b..7e7fa359038 100644 --- a/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java @@ -51,7 +51,9 @@ public class FuzzyQueryParser implements QueryParser { XContentParser parser = parseContext.parser(); XContentParser.Token token = parser.nextToken(); - assert token == XContentParser.Token.FIELD_NAME; + if (token != XContentParser.Token.FIELD_NAME) { + throw new QueryParsingException(parseContext.index(), "[fuzzy] query malformed, no field"); + } String fieldName = parser.currentName(); String value = null; @@ -78,6 +80,8 @@ public class FuzzyQueryParser implements QueryParser { prefixLength = parser.intValue(); } else if ("max_expansions".equals(currentFieldName) || "maxExpansions".equals(currentFieldName)) { maxExpansions = parser.intValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[fuzzy] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterParser.java b/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterParser.java index a17861a33e1..dcb5b87a4e7 100644 --- a/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterParser.java @@ -155,6 +155,8 @@ public class GeoBoundingBoxFilterParser implements FilterParser { normalizeLon = parser.booleanValue(); } else if ("type".equals(currentFieldName)) { type = parser.text(); + } else { + throw new QueryParsingException(parseContext.index(), "[qeo_bbox] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterParser.java b/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterParser.java index f03c5109598..9b05dd988a6 100644 --- a/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterParser.java @@ -44,8 +44,6 @@ import static org.elasticsearch.index.query.support.QueryParsers.wrapSmartNameFi * "name.lon" : 1.2, * } * - * - * */ public class GeoDistanceFilterParser implements FilterParser { @@ -108,6 +106,8 @@ public class GeoDistanceFilterParser implements FilterParser { double[] values = GeoHashUtils.decode(parser.text()); lat = values[0]; lon = values[1]; + } else { + throw new QueryParsingException(parseContext.index(), "[geo_distance] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/GeoPolygonFilterParser.java b/src/main/java/org/elasticsearch/index/query/GeoPolygonFilterParser.java index 620f7c2082d..d8d47ec73b9 100644 --- a/src/main/java/org/elasticsearch/index/query/GeoPolygonFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/GeoPolygonFilterParser.java @@ -49,8 +49,6 @@ import static org.elasticsearch.index.query.support.QueryParsers.wrapSmartNameFi * } * } * - * - * */ public class GeoPolygonFilterParser implements FilterParser { @@ -138,6 +136,8 @@ public class GeoPolygonFilterParser implements FilterParser { points.add(point); } } + } else { + throw new QueryParsingException(parseContext.index(), "[geo_polygon] filter does not support [" + currentFieldName + "]"); } } } @@ -151,6 +151,8 @@ public class GeoPolygonFilterParser implements FilterParser { } else if ("normalize".equals(currentFieldName)) { normalizeLat = parser.booleanValue(); normalizeLon = parser.booleanValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[geo_polygon] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/HasChildFilterParser.java b/src/main/java/org/elasticsearch/index/query/HasChildFilterParser.java index fa9145517b6..a22e4fc3b00 100644 --- a/src/main/java/org/elasticsearch/index/query/HasChildFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/HasChildFilterParser.java @@ -62,12 +62,17 @@ public class HasChildFilterParser implements FilterParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - // since we switch types, make sure we change the context - String[] origTypes = QueryParseContext.setTypesWithPrevious(childType == null ? null : new String[]{childType}); if ("query".equals(currentFieldName)) { - query = parseContext.parseInnerQuery(); + // since we switch types, make sure we change the context + String[] origTypes = QueryParseContext.setTypesWithPrevious(childType == null ? null : new String[]{childType}); + try { + query = parseContext.parseInnerQuery(); + } finally { + QueryParseContext.setTypes(origTypes); + } + } else { + throw new QueryParsingException(parseContext.index(), "[has_child] filter does not support [" + currentFieldName + "]"); } - QueryParseContext.setTypes(origTypes); } else if (token.isValue()) { if ("type".equals(currentFieldName)) { childType = parser.text(); @@ -75,6 +80,8 @@ public class HasChildFilterParser implements FilterParser { scope = parser.text(); } else if ("_name".equals(currentFieldName)) { filterName = parser.text(); + } else { + throw new QueryParsingException(parseContext.index(), "[has_child] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java b/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java index 83eb356b58e..443b02e02a7 100644 --- a/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java @@ -62,12 +62,17 @@ public class HasChildQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - // since we switch types, make sure we change the context - String[] origTypes = QueryParseContext.setTypesWithPrevious(childType == null ? null : new String[]{childType}); if ("query".equals(currentFieldName)) { - query = parseContext.parseInnerQuery(); + // since we switch types, make sure we change the context + String[] origTypes = QueryParseContext.setTypesWithPrevious(childType == null ? null : new String[]{childType}); + try { + query = parseContext.parseInnerQuery(); + } finally { + QueryParseContext.setTypes(origTypes); + } + } else { + throw new QueryParsingException(parseContext.index(), "[has_child] query does not support [" + currentFieldName + "]"); } - QueryParseContext.setTypes(origTypes); } else if (token.isValue()) { if ("type".equals(currentFieldName)) { childType = parser.text(); @@ -75,6 +80,8 @@ public class HasChildQueryParser implements QueryParser { scope = parser.text(); } else if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[has_child] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/IdsFilterParser.java b/src/main/java/org/elasticsearch/index/query/IdsFilterParser.java index e1e6c27aa2a..e90c4a74de3 100644 --- a/src/main/java/org/elasticsearch/index/query/IdsFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/IdsFilterParser.java @@ -76,12 +76,16 @@ public class IdsFilterParser implements FilterParser { } types.add(value); } + } else { + throw new QueryParsingException(parseContext.index(), "[ids] filter does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("type".equals(currentFieldName) || "_type".equals(currentFieldName)) { types = ImmutableList.of(parser.text()); } else if ("_name".equals(currentFieldName)) { filterName = parser.text(); + } else { + throw new QueryParsingException(parseContext.index(), "[ids] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java b/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java index 77566035808..f9548c89577 100644 --- a/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java @@ -80,12 +80,16 @@ public class IdsQueryParser implements QueryParser { } types.add(value); } + } else { + throw new QueryParsingException(parseContext.index(), "[ids] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("type".equals(currentFieldName) || "_type".equals(currentFieldName)) { types = ImmutableList.of(parser.text()); } else if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[ids] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/IndicesQueryParser.java b/src/main/java/org/elasticsearch/index/query/IndicesQueryParser.java index 64fe629ee72..0cf6928e36a 100644 --- a/src/main/java/org/elasticsearch/index/query/IndicesQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/IndicesQueryParser.java @@ -63,6 +63,8 @@ public class IndicesQueryParser implements QueryParser { query = parseContext.parseInnerQuery(); } else if ("no_match_query".equals(currentFieldName)) { noMatchQuery = parseContext.parseInnerQuery(); + } else { + throw new QueryParsingException(parseContext.index(), "[indices] query does not support [" + currentFieldName + "]"); } } else if (token == XContentParser.Token.START_ARRAY) { if ("indices".equals(currentFieldName)) { @@ -73,6 +75,8 @@ public class IndicesQueryParser implements QueryParser { } indices.add(value); } + } else { + throw new QueryParsingException(parseContext.index(), "[indices] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("index".equals(currentFieldName)) { @@ -84,6 +88,8 @@ public class IndicesQueryParser implements QueryParser { } else if ("none".equals(type)) { noMatchQuery = MatchNoDocsQuery.INSTANCE; } + } else { + throw new QueryParsingException(parseContext.index(), "[indices] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/LimitFilterParser.java b/src/main/java/org/elasticsearch/index/query/LimitFilterParser.java index ea1001a613e..c22e16c14cb 100644 --- a/src/main/java/org/elasticsearch/index/query/LimitFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/LimitFilterParser.java @@ -52,6 +52,8 @@ public class LimitFilterParser implements FilterParser { } else if (token.isValue()) { if ("value".equals(currentFieldName)) { limit = parser.intValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[limit] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/MatchAllQueryParser.java b/src/main/java/org/elasticsearch/index/query/MatchAllQueryParser.java index ab8dde26c88..21621d30c5c 100644 --- a/src/main/java/org/elasticsearch/index/query/MatchAllQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/MatchAllQueryParser.java @@ -61,6 +61,8 @@ public class MatchAllQueryParser implements QueryParser { boost = parser.floatValue(); } else if ("norms_field".equals(currentFieldName) || "normsField".equals(currentFieldName)) { normsField = parseContext.indexName(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[match_all] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java b/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java index 2bddd1163da..17978ce5f67 100644 --- a/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java @@ -63,6 +63,8 @@ public class MissingFilterParser implements FilterParser { fieldName = parser.text(); } else if ("_name".equals(currentFieldName)) { filterName = parser.text(); + } else { + throw new QueryParsingException(parseContext.index(), "[missing] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/MoreLikeThisFieldQueryParser.java b/src/main/java/org/elasticsearch/index/query/MoreLikeThisFieldQueryParser.java index 1eddb00cb99..69c95455f4f 100644 --- a/src/main/java/org/elasticsearch/index/query/MoreLikeThisFieldQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/MoreLikeThisFieldQueryParser.java @@ -92,6 +92,8 @@ public class MoreLikeThisFieldQueryParser implements QueryParser { mltQuery.setPercentTermsToMatch(parser.floatValue()); } else if ("analyzer".equals(currentFieldName)) { analyzer = parseContext.analysisService().analyzer(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[mlt_field] query does not support [" + currentFieldName + "]"); } } else if (token == XContentParser.Token.START_ARRAY) { if ("stop_words".equals(currentFieldName) || "stopWords".equals(currentFieldName)) { @@ -100,6 +102,8 @@ public class MoreLikeThisFieldQueryParser implements QueryParser { stopWords.add(parser.text()); } mltQuery.setStopWords(stopWords); + } else { + throw new QueryParsingException(parseContext.index(), "[mlt_field] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryParser.java b/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryParser.java index 34241bdbbcd..63e2269da0f 100644 --- a/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryParser.java @@ -84,6 +84,8 @@ public class MoreLikeThisQueryParser implements QueryParser { mltQuery.setPercentTermsToMatch(parser.floatValue()); } else if ("analyzer".equals(currentFieldName)) { analyzer = parseContext.analysisService().analyzer(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[mlt] query does not support [" + currentFieldName + "]"); } } else if (token == XContentParser.Token.START_ARRAY) { if ("stop_words".equals(currentFieldName) || "stopWords".equals(currentFieldName)) { @@ -98,6 +100,8 @@ public class MoreLikeThisQueryParser implements QueryParser { fields.add(parseContext.indexName(parser.text())); } mltQuery.setMoreLikeFields(fields.toArray(new String[fields.size()])); + } else { + throw new QueryParsingException(parseContext.index(), "[mlt] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/NestedFilterParser.java b/src/main/java/org/elasticsearch/index/query/NestedFilterParser.java index 7da8860da52..30438c5c59a 100644 --- a/src/main/java/org/elasticsearch/index/query/NestedFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/NestedFilterParser.java @@ -75,6 +75,8 @@ public class NestedFilterParser implements FilterParser { query = parseContext.parseInnerQuery(); } else if ("filter".equals(currentFieldName)) { filter = parseContext.parseInnerFilter(); + } else { + throw new QueryParsingException(parseContext.index(), "[nested] filter does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("path".equals(currentFieldName)) { @@ -89,6 +91,8 @@ public class NestedFilterParser implements FilterParser { cache = parser.booleanValue(); } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { cacheKey = new CacheKeyFilter.Key(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[nested] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java b/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java index da30dcd7bac..e8fb0b17d3a 100644 --- a/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java @@ -73,6 +73,8 @@ public class NestedQueryParser implements QueryParser { query = parseContext.parseInnerQuery(); } else if ("filter".equals(currentFieldName)) { filter = parseContext.parseInnerFilter(); + } else { + throw new QueryParsingException(parseContext.index(), "[nested] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("path".equals(currentFieldName)) { @@ -94,6 +96,8 @@ public class NestedQueryParser implements QueryParser { } else { throw new QueryParsingException(parseContext.index(), "illegal score_mode for nested query [" + sScoreMode + "]"); } + } else { + throw new QueryParsingException(parseContext.index(), "[nested] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/NotFilterParser.java b/src/main/java/org/elasticsearch/index/query/NotFilterParser.java index 1b0dcf14877..bc05ba6e13f 100644 --- a/src/main/java/org/elasticsearch/index/query/NotFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/NotFilterParser.java @@ -71,6 +71,8 @@ public class NotFilterParser implements FilterParser { filterName = parser.text(); } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { cacheKey = new CacheKeyFilter.Key(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[not] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/NumericRangeFilterParser.java b/src/main/java/org/elasticsearch/index/query/NumericRangeFilterParser.java index 0a0e24f9524..d2037f933ce 100644 --- a/src/main/java/org/elasticsearch/index/query/NumericRangeFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/NumericRangeFilterParser.java @@ -91,6 +91,8 @@ public class NumericRangeFilterParser implements FilterParser { } else if ("lte".equals(currentFieldName) || "le".equals(currentFieldName)) { to = parser.textOrNull(); includeUpper = true; + } else { + throw new QueryParsingException(parseContext.index(), "[numeric_range] filter does not support [" + currentFieldName + "]"); } } } @@ -101,6 +103,8 @@ public class NumericRangeFilterParser implements FilterParser { cache = parser.booleanValue(); } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { cacheKey = new CacheKeyFilter.Key(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[numeric_range] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/OrFilterParser.java b/src/main/java/org/elasticsearch/index/query/OrFilterParser.java index c1d6f0edf88..892f3c5f3e3 100644 --- a/src/main/java/org/elasticsearch/index/query/OrFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/OrFilterParser.java @@ -83,6 +83,8 @@ public class OrFilterParser implements FilterParser { filterName = parser.text(); } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { cacheKey = new CacheKeyFilter.Key(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[or] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java b/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java index b8d33761765..ef17de274bb 100644 --- a/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java @@ -52,7 +52,9 @@ public class PrefixQueryParser implements QueryParser { XContentParser parser = parseContext.parser(); XContentParser.Token token = parser.nextToken(); - assert token == XContentParser.Token.FIELD_NAME; + if (token != XContentParser.Token.FIELD_NAME) { + throw new QueryParsingException(parseContext.index(), "[prefix] query malformed, no field"); + } String fieldName = parser.currentName(); String rewriteMethod = null; @@ -74,6 +76,8 @@ public class PrefixQueryParser implements QueryParser { } else if ("rewrite".equals(currentFieldName)) { rewriteMethod = parser.textOrNull(); } + } else { + throw new QueryParsingException(parseContext.index(), "[prefix] query does not support [" + currentFieldName + "]"); } } parser.nextToken(); diff --git a/src/main/java/org/elasticsearch/index/query/QueryParseContext.java b/src/main/java/org/elasticsearch/index/query/QueryParseContext.java index 48fadc0450a..37d470e3d93 100644 --- a/src/main/java/org/elasticsearch/index/query/QueryParseContext.java +++ b/src/main/java/org/elasticsearch/index/query/QueryParseContext.java @@ -161,14 +161,20 @@ public class QueryParseContext { XContentParser.Token token; if (parser.currentToken() != XContentParser.Token.START_OBJECT) { token = parser.nextToken(); - assert token == XContentParser.Token.START_OBJECT; + if (token != XContentParser.Token.START_OBJECT) { + throw new QueryParsingException(index, "[_na] query malformed, must start with start_object"); + } } token = parser.nextToken(); - assert token == XContentParser.Token.FIELD_NAME; + if (token != XContentParser.Token.FIELD_NAME) { + throw new QueryParsingException(index, "[_na] query malformed, no field after start_object"); + } String queryName = parser.currentName(); // move to the next START_OBJECT token = parser.nextToken(); - assert token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.START_ARRAY; + if (token != XContentParser.Token.START_OBJECT && token != XContentParser.Token.START_ARRAY) { + throw new QueryParsingException(index, "[_na] query malformed, no field after start_object"); + } QueryParser queryParser = indexQueryParser.queryParser(queryName); if (queryParser == null) { @@ -187,14 +193,20 @@ public class QueryParseContext { XContentParser.Token token; if (parser.currentToken() != XContentParser.Token.START_OBJECT) { token = parser.nextToken(); - assert token == XContentParser.Token.START_OBJECT; + if (token != XContentParser.Token.START_OBJECT) { + throw new QueryParsingException(index, "[_na] filter malformed, must start with start_object"); + } } token = parser.nextToken(); - assert token == XContentParser.Token.FIELD_NAME; + if (token != XContentParser.Token.FIELD_NAME) { + throw new QueryParsingException(index, "[_na] filter malformed, no field after start_object"); + } String filterName = parser.currentName(); // move to the next START_OBJECT or START_ARRAY token = parser.nextToken(); - assert token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.START_ARRAY; + if (token != XContentParser.Token.START_OBJECT && token != XContentParser.Token.START_ARRAY) { + throw new QueryParsingException(index, "[_na] filter malformed, no field after start_object"); + } FilterParser filterParser = indexQueryParser.filterParser(filterName); if (filterParser == null) { diff --git a/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java b/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java index bb2d48f1e36..04b55bfea74 100644 --- a/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java @@ -119,6 +119,8 @@ public class QueryStringQueryParser implements QueryParser { } } } + } else { + throw new QueryParsingException(parseContext.index(), "[query_string] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("query".equals(currentFieldName)) { @@ -164,6 +166,8 @@ public class QueryStringQueryParser implements QueryParser { qpSettings.rewriteMethod(QueryParsers.parseRewriteMethod(parser.textOrNull())); } else if ("minimum_should_match".equals(currentFieldName) || "minimumShouldMatch".equals(currentFieldName)) { qpSettings.minimumShouldMatch(parser.textOrNull()); + } else { + throw new QueryParsingException(parseContext.index(), "[query_string] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/RangeFilterParser.java b/src/main/java/org/elasticsearch/index/query/RangeFilterParser.java index 4871f94b490..e8726545b58 100644 --- a/src/main/java/org/elasticsearch/index/query/RangeFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/RangeFilterParser.java @@ -90,6 +90,8 @@ public class RangeFilterParser implements FilterParser { } else if ("lte".equals(currentFieldName) || "le".equals(currentFieldName)) { to = parser.textOrNull(); includeUpper = true; + } else { + throw new QueryParsingException(parseContext.index(), "[range] filter does not support [" + currentFieldName + "]"); } } } @@ -100,6 +102,8 @@ public class RangeFilterParser implements FilterParser { cache = parser.booleanValue(); } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { cacheKey = new CacheKeyFilter.Key(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[range] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java b/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java index 14c678491ce..dc1a9c7f4bb 100644 --- a/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java @@ -50,8 +50,14 @@ public class RangeQueryParser implements QueryParser { XContentParser parser = parseContext.parser(); XContentParser.Token token = parser.nextToken(); - assert token == XContentParser.Token.FIELD_NAME; + if (token != XContentParser.Token.FIELD_NAME) { + throw new QueryParsingException(parseContext.index(), "[range] query malformed, no field to indicate field name"); + } String fieldName = parser.currentName(); + token = parser.nextToken(); + if (token != XContentParser.Token.START_OBJECT) { + throw new QueryParsingException(parseContext.index(), "[range] query malformed, after field missing start object"); + } String from = null; String to = null; @@ -86,13 +92,17 @@ public class RangeQueryParser implements QueryParser { } else if ("lte".equals(currentFieldName) || "le".equals(currentFieldName)) { to = parser.textOrNull(); includeUpper = true; + } else { + throw new QueryParsingException(parseContext.index(), "[range] query does not support [" + currentFieldName + "]"); } } } // move to the next end object, to close the field name token = parser.nextToken(); - assert token == XContentParser.Token.END_OBJECT; + if (token != XContentParser.Token.END_OBJECT) { + throw new QueryParsingException(parseContext.index(), "[range] query malformed, does not end with an object"); + } Query query = null; MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); diff --git a/src/main/java/org/elasticsearch/index/query/ScriptFilterParser.java b/src/main/java/org/elasticsearch/index/query/ScriptFilterParser.java index e9820711ea5..d77557d487d 100644 --- a/src/main/java/org/elasticsearch/index/query/ScriptFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/ScriptFilterParser.java @@ -73,6 +73,8 @@ public class ScriptFilterParser implements FilterParser { } else if (token == XContentParser.Token.START_OBJECT) { if ("params".equals(currentFieldName)) { params = parser.map(); + } else { + throw new QueryParsingException(parseContext.index(), "[script] filter does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("script".equals(currentFieldName)) { @@ -85,6 +87,8 @@ public class ScriptFilterParser implements FilterParser { cache = parser.booleanValue(); } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { cacheKey = new CacheKeyFilter.Key(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[script] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/SpanFirstQueryParser.java b/src/main/java/org/elasticsearch/index/query/SpanFirstQueryParser.java index 3dcba37c86a..b30641bf44f 100644 --- a/src/main/java/org/elasticsearch/index/query/SpanFirstQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/SpanFirstQueryParser.java @@ -65,12 +65,16 @@ public class SpanFirstQueryParser implements QueryParser { throw new QueryParsingException(parseContext.index(), "spanFirst [match] must be of type span query"); } match = (SpanQuery) query; + } else { + throw new QueryParsingException(parseContext.index(), "[span_first] query does not support [" + currentFieldName + "]"); } } else { if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); } else if ("end".equals(currentFieldName)) { end = parser.intValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[span_first] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/SpanNearQueryParser.java b/src/main/java/org/elasticsearch/index/query/SpanNearQueryParser.java index 2d7afbd651f..2022f084967 100644 --- a/src/main/java/org/elasticsearch/index/query/SpanNearQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/SpanNearQueryParser.java @@ -72,6 +72,8 @@ public class SpanNearQueryParser implements QueryParser { } clauses.add((SpanQuery) query); } + } else { + throw new QueryParsingException(parseContext.index(), "[span_near] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("in_order".equals(currentFieldName) || "inOrder".equals(currentFieldName)) { @@ -82,7 +84,11 @@ public class SpanNearQueryParser implements QueryParser { slop = parser.intValue(); } else if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[span_near] query does not support [" + currentFieldName + "]"); } + } else { + throw new QueryParsingException(parseContext.index(), "[span_near] query does not support [" + currentFieldName + "]"); } } if (clauses.isEmpty()) { diff --git a/src/main/java/org/elasticsearch/index/query/SpanNotQueryParser.java b/src/main/java/org/elasticsearch/index/query/SpanNotQueryParser.java index 6091f72bc00..c767dfb0777 100644 --- a/src/main/java/org/elasticsearch/index/query/SpanNotQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/SpanNotQueryParser.java @@ -71,10 +71,14 @@ public class SpanNotQueryParser implements QueryParser { throw new QueryParsingException(parseContext.index(), "spanNot [exclude] must be of type span query"); } exclude = (SpanQuery) query; + } else { + throw new QueryParsingException(parseContext.index(), "[span_not] query does not support [" + currentFieldName + "]"); } } else { if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[span_not] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/SpanOrQueryParser.java b/src/main/java/org/elasticsearch/index/query/SpanOrQueryParser.java index d3d1599ea3b..91023d50bb7 100644 --- a/src/main/java/org/elasticsearch/index/query/SpanOrQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/SpanOrQueryParser.java @@ -69,10 +69,14 @@ public class SpanOrQueryParser implements QueryParser { } clauses.add((SpanQuery) query); } + } else { + throw new QueryParsingException(parseContext.index(), "[span_or] query does not support [" + currentFieldName + "]"); } } else { if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[span_or] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/SpanTermQueryParser.java b/src/main/java/org/elasticsearch/index/query/SpanTermQueryParser.java index e8ec4118952..c059c85470e 100644 --- a/src/main/java/org/elasticsearch/index/query/SpanTermQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/SpanTermQueryParser.java @@ -74,6 +74,8 @@ public class SpanTermQueryParser implements QueryParser { value = parser.text(); } else if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[span_term] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/TermQueryParser.java b/src/main/java/org/elasticsearch/index/query/TermQueryParser.java index 3882470ec4b..c5f2c3e9077 100644 --- a/src/main/java/org/elasticsearch/index/query/TermQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/TermQueryParser.java @@ -51,7 +51,9 @@ public class TermQueryParser implements QueryParser { XContentParser parser = parseContext.parser(); XContentParser.Token token = parser.nextToken(); - assert token == XContentParser.Token.FIELD_NAME; + if (token != XContentParser.Token.FIELD_NAME) { + throw new QueryParsingException(parseContext.index(), "[term] query malformed, no field"); + } String fieldName = parser.currentName(); String value = null; @@ -69,6 +71,8 @@ public class TermQueryParser implements QueryParser { value = parser.text(); } else if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[term] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java b/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java index 0bb50ec9d8b..3698c128a98 100644 --- a/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java @@ -88,6 +88,8 @@ public class TermsFilterParser implements FilterParser { cache = parser.booleanValue(); } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { cacheKey = new CacheKeyFilter.Key(parser.text()); + } else { + throw new QueryParsingException(parseContext.index(), "[terms] filter does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java b/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java index 41960f8e6cd..721a919fd00 100644 --- a/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java @@ -44,8 +44,6 @@ import static org.elasticsearch.index.query.support.QueryParsers.wrapSmartNameQu * "minimum_match" : 1 * } * - * - * */ public class TermsQueryParser implements QueryParser { @@ -92,6 +90,8 @@ public class TermsQueryParser implements QueryParser { } else if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); } + } else { + throw new QueryParsingException(parseContext.index(), "[terms] query does not support [" + currentFieldName + "]"); } } diff --git a/src/main/java/org/elasticsearch/index/query/TextQueryParser.java b/src/main/java/org/elasticsearch/index/query/TextQueryParser.java index f8a5551aae0..b8042b49029 100644 --- a/src/main/java/org/elasticsearch/index/query/TextQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/TextQueryParser.java @@ -55,7 +55,9 @@ public class TextQueryParser implements QueryParser { } XContentParser.Token token = parser.nextToken(); - assert token == XContentParser.Token.FIELD_NAME; + if (token != XContentParser.Token.FIELD_NAME) { + throw new QueryParsingException(parseContext.index(), "[text] query malformed, no field"); + } String fieldName = parser.currentName(); String text = null; @@ -106,6 +108,8 @@ public class TextQueryParser implements QueryParser { } else { throw new QueryParsingException(parseContext.index(), "text query requires operator to be either 'and' or 'or', not [" + op + "]"); } + } else { + throw new QueryParsingException(parseContext.index(), "[text] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/TopChildrenQueryParser.java b/src/main/java/org/elasticsearch/index/query/TopChildrenQueryParser.java index 87e221bc65b..98ad75e2e1a 100644 --- a/src/main/java/org/elasticsearch/index/query/TopChildrenQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/TopChildrenQueryParser.java @@ -66,6 +66,8 @@ public class TopChildrenQueryParser implements QueryParser { } else if (token == XContentParser.Token.START_OBJECT) { if ("query".equals(currentFieldName)) { query = parseContext.parseInnerQuery(); + } else { + throw new QueryParsingException(parseContext.index(), "[top_children] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { if ("type".equals(currentFieldName)) { @@ -80,6 +82,8 @@ public class TopChildrenQueryParser implements QueryParser { factor = parser.intValue(); } else if ("incremental_factor".equals(currentFieldName) || "incrementalFactor".equals(currentFieldName)) { incrementalFactor = parser.intValue(); + } else { + throw new QueryParsingException(parseContext.index(), "[top_children] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/TypeFilterParser.java b/src/main/java/org/elasticsearch/index/query/TypeFilterParser.java index 76a224a03a5..b4f0a0a4bc4 100644 --- a/src/main/java/org/elasticsearch/index/query/TypeFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/TypeFilterParser.java @@ -47,15 +47,15 @@ public class TypeFilterParser implements FilterParser { XContentParser.Token token = parser.nextToken(); if (token != XContentParser.Token.FIELD_NAME) { - throw new QueryParsingException(parseContext.index(), "type filter should have a value field, and the type name"); + throw new QueryParsingException(parseContext.index(), "[type] filter should have a value field, and the type name"); } String fieldName = parser.currentName(); if (!fieldName.equals("value")) { - throw new QueryParsingException(parseContext.index(), "type filter should have a value field, and the type name"); + throw new QueryParsingException(parseContext.index(), "[type] filter should have a value field, and the type name"); } token = parser.nextToken(); if (token != XContentParser.Token.VALUE_STRING) { - throw new QueryParsingException(parseContext.index(), "type filter should have a value field, and the type name"); + throw new QueryParsingException(parseContext.index(), "[type] filter should have a value field, and the type name"); } String type = parser.text(); // move to the next token diff --git a/src/main/java/org/elasticsearch/index/query/WildcardQueryParser.java b/src/main/java/org/elasticsearch/index/query/WildcardQueryParser.java index 7a790be9d01..bdad45a6161 100644 --- a/src/main/java/org/elasticsearch/index/query/WildcardQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/WildcardQueryParser.java @@ -52,7 +52,9 @@ public class WildcardQueryParser implements QueryParser { XContentParser parser = parseContext.parser(); XContentParser.Token token = parser.nextToken(); - assert token == XContentParser.Token.FIELD_NAME; + if (token != XContentParser.Token.FIELD_NAME) { + throw new QueryParsingException(parseContext.index(), "[wildcard] query malformed, no field"); + } String fieldName = parser.currentName(); String rewriteMethod = null; @@ -73,6 +75,8 @@ public class WildcardQueryParser implements QueryParser { boost = parser.floatValue(); } else if ("rewrite".equals(currentFieldName)) { rewriteMethod = parser.textOrNull(); + } else { + throw new QueryParsingException(parseContext.index(), "[wildcard] query does not support [" + currentFieldName + "]"); } } } diff --git a/src/main/java/org/elasticsearch/index/query/WrapperQueryParser.java b/src/main/java/org/elasticsearch/index/query/WrapperQueryParser.java index 7783fc48c74..44a1de87fba 100644 --- a/src/main/java/org/elasticsearch/index/query/WrapperQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/WrapperQueryParser.java @@ -47,9 +47,13 @@ public class WrapperQueryParser implements QueryParser { XContentParser parser = parseContext.parser(); XContentParser.Token token = parser.nextToken(); - assert token == XContentParser.Token.FIELD_NAME; + if (token != XContentParser.Token.FIELD_NAME) { + throw new QueryParsingException(parseContext.index(), "[wrapper] query malformed"); + } String fieldName = parser.currentName(); - assert fieldName.equals("query"); + if (!fieldName.equals("query")) { + throw new QueryParsingException(parseContext.index(), "[wrapper] query malformed"); + } parser.nextToken(); byte[] querySource = parser.binaryValue(); From 0922b5e80cf2689381e9382a3aefd0f136136a05 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 28 Jan 2012 00:25:26 +0200 Subject: [PATCH 209/270] improve type based query when using ids (or term on _id) perf --- .../lucene/queryParser/MapperQueryParser.java | 11 +- .../index/mapper/internal/IdFieldMapper.java | 4 +- .../index/query/IdsFilterParser.java | 9 +- .../index/query/IdsQueryParser.java | 9 +- .../index/query/QueryParseContext.java | 13 ++ .../index/query/TermFilterParser.java | 11 +- .../index/query/TermQueryParser.java | 11 +- .../index/query/TermsFilterParser.java | 184 +++++++++--------- .../index/search/TextQueryParser.java | 11 +- .../shard/service/InternalIndexShard.java | 8 +- 10 files changed, 161 insertions(+), 110 deletions(-) diff --git a/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java b/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java index a52194af67f..4a5d98c4caa 100644 --- a/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java +++ b/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java @@ -131,7 +131,16 @@ public class MapperQueryParser extends QueryParser { if (currentMapper != null) { Query query = null; if (currentMapper.useFieldQueryWithQueryString()) { - query = currentMapper.fieldQuery(queryText, parseContext); + if (fieldMappers.hasDocMapper()) { + String[] previousTypes = QueryParseContext.setTypesWithPrevious(new String[]{fieldMappers.docMapper().type()}); + try { + query = currentMapper.fieldQuery(queryText, parseContext); + } finally { + QueryParseContext.setTypes(previousTypes); + } + } else { + query = currentMapper.fieldQuery(queryText, parseContext); + } } if (query == null) { query = super.getFieldQuery(currentMapper.names().indexName(), queryText, quoted); diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java index 04c1e0ee5c7..60b7f4b9bc6 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java @@ -162,7 +162,7 @@ public class IdFieldMapper extends AbstractFieldMapper implements Intern if (indexed() || context == null) { return super.fieldQuery(value, context); } - UidFilter filter = new UidFilter(context.mapperService().types(), ImmutableList.of(value), context.indexCache().bloomCache()); + UidFilter filter = new UidFilter(context.queryTypes(), ImmutableList.of(value), context.indexCache().bloomCache()); // no need for constant score filter, since we don't cache the filter, and it always takes deletes into account return new ConstantScoreQuery(filter); } @@ -172,7 +172,7 @@ public class IdFieldMapper extends AbstractFieldMapper implements Intern if (indexed() || context == null) { return super.fieldFilter(value, context); } - return new UidFilter(context.mapperService().types(), ImmutableList.of(value), context.indexCache().bloomCache()); + return new UidFilter(context.queryTypes(), ImmutableList.of(value), context.indexCache().bloomCache()); } @Override diff --git a/src/main/java/org/elasticsearch/index/query/IdsFilterParser.java b/src/main/java/org/elasticsearch/index/query/IdsFilterParser.java index e90c4a74de3..59e341e3a3e 100644 --- a/src/main/java/org/elasticsearch/index/query/IdsFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/IdsFilterParser.java @@ -25,11 +25,9 @@ import org.apache.lucene.search.Filter; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.search.UidFilter; -import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -95,12 +93,7 @@ public class IdsFilterParser implements FilterParser { } if (types == null || types.isEmpty()) { - SearchContext searchContext = SearchContext.current(); - if (searchContext.hasTypes()) { - types = Arrays.asList(searchContext.types()); - } else { - types = parseContext.mapperService().types(); - } + types = parseContext.queryTypes(); } else if (types.size() == 1 && Iterables.getFirst(types, null).equals("_all")) { types = parseContext.mapperService().types(); } diff --git a/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java b/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java index f9548c89577..3c4bd5264c4 100644 --- a/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java @@ -26,11 +26,9 @@ import org.apache.lucene.search.Query; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.search.UidFilter; -import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -99,12 +97,7 @@ public class IdsQueryParser implements QueryParser { } if (types == null || types.isEmpty()) { - SearchContext searchContext = SearchContext.current(); - if (searchContext.hasTypes()) { - types = Arrays.asList(searchContext.types()); - } else { - types = parseContext.mapperService().types(); - } + types = parseContext.queryTypes(); } else if (types.size() == 1 && Iterables.getFirst(types, null).equals("_all")) { types = parseContext.mapperService().types(); } diff --git a/src/main/java/org/elasticsearch/index/query/QueryParseContext.java b/src/main/java/org/elasticsearch/index/query/QueryParseContext.java index 37d470e3d93..f2902a844bd 100644 --- a/src/main/java/org/elasticsearch/index/query/QueryParseContext.java +++ b/src/main/java/org/elasticsearch/index/query/QueryParseContext.java @@ -42,6 +42,8 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.script.ScriptService; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; import java.util.Map; /** @@ -257,4 +259,15 @@ public class QueryParseContext { public MapperService.SmartNameObjectMapper smartObjectMapper(String name) { return indexQueryParser.mapperService.smartNameObjectMapper(name, getTypes()); } + + /** + * Returns the narrowed down explicit types, or, if not set, all types. + */ + public Collection queryTypes() { + String[] types = getTypes(); + if (types == null || types.length == 0) { + return mapperService().types(); + } + return Arrays.asList(types); + } } diff --git a/src/main/java/org/elasticsearch/index/query/TermFilterParser.java b/src/main/java/org/elasticsearch/index/query/TermFilterParser.java index c764a984fa5..7c3257bd69e 100644 --- a/src/main/java/org/elasticsearch/index/query/TermFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/TermFilterParser.java @@ -88,7 +88,16 @@ public class TermFilterParser implements FilterParser { MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); if (smartNameFieldMappers != null) { if (smartNameFieldMappers.hasMapper()) { - filter = smartNameFieldMappers.mapper().fieldFilter(value, parseContext); + if (smartNameFieldMappers.hasDocMapper()) { + String[] previousTypes = QueryParseContext.setTypesWithPrevious(new String[]{smartNameFieldMappers.docMapper().type()}); + try { + filter = smartNameFieldMappers.mapper().fieldFilter(value, parseContext); + } finally { + QueryParseContext.setTypes(previousTypes); + } + } else { + filter = smartNameFieldMappers.mapper().fieldFilter(value, parseContext); + } } } if (filter == null) { diff --git a/src/main/java/org/elasticsearch/index/query/TermQueryParser.java b/src/main/java/org/elasticsearch/index/query/TermQueryParser.java index c5f2c3e9077..8de9d6df915 100644 --- a/src/main/java/org/elasticsearch/index/query/TermQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/TermQueryParser.java @@ -91,7 +91,16 @@ public class TermQueryParser implements QueryParser { MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); if (smartNameFieldMappers != null) { if (smartNameFieldMappers.hasMapper()) { - query = smartNameFieldMappers.mapper().fieldQuery(value, parseContext); + if (smartNameFieldMappers.hasDocMapper()) { + String[] previousTypes = QueryParseContext.setTypesWithPrevious(new String[]{smartNameFieldMappers.docMapper().type()}); + try { + query = smartNameFieldMappers.mapper().fieldQuery(value, parseContext); + } finally { + QueryParseContext.setTypes(previousTypes); + } + } else { + query = smartNameFieldMappers.mapper().fieldQuery(value, parseContext); + } } } if (query == null) { diff --git a/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java b/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java index 3698c128a98..53917b9a096 100644 --- a/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java @@ -100,103 +100,113 @@ public class TermsFilterParser implements FilterParser { FieldMapper fieldMapper = null; smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); + String[] previousTypes = null; if (smartNameFieldMappers != null) { if (smartNameFieldMappers.hasMapper()) { fieldMapper = smartNameFieldMappers.mapper(); fieldName = fieldMapper.names().indexName(); } + // if we have a doc mapper, its explicit type, mark it + if (smartNameFieldMappers.hasDocMapper()) { + previousTypes = QueryParseContext.setTypesWithPrevious(new String[]{smartNameFieldMappers.docMapper().type()}); + } } + try { + Filter filter; + if ("plain".equals(execution)) { + PublicTermsFilter termsFilter = new PublicTermsFilter(); + if (fieldMapper != null) { + for (String term : terms) { + termsFilter.addTerm(fieldMapper.names().createIndexNameTerm(fieldMapper.indexedValue(term))); + } + } else { + for (String term : terms) { + termsFilter.addTerm(new Term(fieldName, term)); + } + } + filter = termsFilter; + // cache the whole filter by default, or if explicitly told to + if (cache == null || cache) { + filter = parseContext.cacheFilter(filter, cacheKey); + } + } else if ("bool".equals(execution)) { + XBooleanFilter boolFiler = new XBooleanFilter(); + if (fieldMapper != null) { + for (String term : terms) { + boolFiler.addShould(parseContext.cacheFilter(fieldMapper.fieldFilter(term, parseContext), null)); + } + } else { + for (String term : terms) { + boolFiler.addShould(parseContext.cacheFilter(new TermFilter(new Term(fieldName, term)), null)); + } + } + filter = boolFiler; + // only cache if explicitly told to, since we cache inner filters + if (cache != null && cache) { + filter = parseContext.cacheFilter(filter, cacheKey); + } + } else if ("bool_nocache".equals(execution)) { + XBooleanFilter boolFiler = new XBooleanFilter(); + if (fieldMapper != null) { + for (String term : terms) { + boolFiler.addShould(fieldMapper.fieldFilter(term, parseContext)); + } + } else { + for (String term : terms) { + boolFiler.addShould(new TermFilter(new Term(fieldName, term))); + } + } + filter = boolFiler; + // cache the whole filter by default, or if explicitly told to + if (cache == null || cache) { + filter = parseContext.cacheFilter(filter, cacheKey); + } + } else if ("and".equals(execution)) { + List filters = Lists.newArrayList(); + if (fieldMapper != null) { + for (String term : terms) { + filters.add(parseContext.cacheFilter(fieldMapper.fieldFilter(term, parseContext), null)); + } + } else { + for (String term : terms) { + filters.add(parseContext.cacheFilter(new TermFilter(new Term(fieldName, term)), null)); + } + } + filter = new AndFilter(filters); + // only cache if explicitly told to, since we cache inner filters + if (cache != null && cache) { + filter = parseContext.cacheFilter(filter, cacheKey); + } + } else if ("and_nocache".equals(execution)) { + List filters = Lists.newArrayList(); + if (fieldMapper != null) { + for (String term : terms) { + filters.add(fieldMapper.fieldFilter(term, parseContext)); + } + } else { + for (String term : terms) { + filters.add(new TermFilter(new Term(fieldName, term))); + } + } + filter = new AndFilter(filters); + // cache the whole filter by default, or if explicitly told to + if (cache == null || cache) { + filter = parseContext.cacheFilter(filter, cacheKey); + } + } else { + throw new QueryParsingException(parseContext.index(), "bool filter execution value [" + execution + "] not supported"); + } - Filter filter; - if ("plain".equals(execution)) { - PublicTermsFilter termsFilter = new PublicTermsFilter(); - if (fieldMapper != null) { - for (String term : terms) { - termsFilter.addTerm(fieldMapper.names().createIndexNameTerm(fieldMapper.indexedValue(term))); - } - } else { - for (String term : terms) { - termsFilter.addTerm(new Term(fieldName, term)); - } + filter = wrapSmartNameFilter(filter, smartNameFieldMappers, parseContext); + if (filterName != null) { + parseContext.addNamedFilter(filterName, filter); } - filter = termsFilter; - // cache the whole filter by default, or if explicitly told to - if (cache == null || cache) { - filter = parseContext.cacheFilter(filter, cacheKey); + return filter; + } finally { + if (smartNameFieldMappers != null && smartNameFieldMappers.hasDocMapper()) { + QueryParseContext.setTypes(previousTypes); } - } else if ("bool".equals(execution)) { - XBooleanFilter boolFiler = new XBooleanFilter(); - if (fieldMapper != null) { - for (String term : terms) { - boolFiler.addShould(parseContext.cacheFilter(fieldMapper.fieldFilter(term, parseContext), null)); - } - } else { - for (String term : terms) { - boolFiler.addShould(parseContext.cacheFilter(new TermFilter(new Term(fieldName, term)), null)); - } - } - filter = boolFiler; - // only cache if explicitly told to, since we cache inner filters - if (cache != null && cache) { - filter = parseContext.cacheFilter(filter, cacheKey); - } - } else if ("bool_nocache".equals(execution)) { - XBooleanFilter boolFiler = new XBooleanFilter(); - if (fieldMapper != null) { - for (String term : terms) { - boolFiler.addShould(fieldMapper.fieldFilter(term, parseContext)); - } - } else { - for (String term : terms) { - boolFiler.addShould(new TermFilter(new Term(fieldName, term))); - } - } - filter = boolFiler; - // cache the whole filter by default, or if explicitly told to - if (cache == null || cache) { - filter = parseContext.cacheFilter(filter, cacheKey); - } - } else if ("and".equals(execution)) { - List filters = Lists.newArrayList(); - if (fieldMapper != null) { - for (String term : terms) { - filters.add(parseContext.cacheFilter(fieldMapper.fieldFilter(term, parseContext), null)); - } - } else { - for (String term : terms) { - filters.add(parseContext.cacheFilter(new TermFilter(new Term(fieldName, term)), null)); - } - } - filter = new AndFilter(filters); - // only cache if explicitly told to, since we cache inner filters - if (cache != null && cache) { - filter = parseContext.cacheFilter(filter, cacheKey); - } - } else if ("and_nocache".equals(execution)) { - List filters = Lists.newArrayList(); - if (fieldMapper != null) { - for (String term : terms) { - filters.add(fieldMapper.fieldFilter(term, parseContext)); - } - } else { - for (String term : terms) { - filters.add(new TermFilter(new Term(fieldName, term))); - } - } - filter = new AndFilter(filters); - // cache the whole filter by default, or if explicitly told to - if (cache == null || cache) { - filter = parseContext.cacheFilter(filter, cacheKey); - } - } else { - throw new QueryParsingException(parseContext.index(), "bool filter execution value [" + execution + "] not supported"); } - - filter = wrapSmartNameFilter(filter, smartNameFieldMappers, parseContext); - if (filterName != null) { - parseContext.addNamedFilter(filterName, filter); - } - return filter; } } diff --git a/src/main/java/org/elasticsearch/index/search/TextQueryParser.java b/src/main/java/org/elasticsearch/index/search/TextQueryParser.java index 90b759ac25f..4ecdd59bd1e 100644 --- a/src/main/java/org/elasticsearch/index/search/TextQueryParser.java +++ b/src/main/java/org/elasticsearch/index/search/TextQueryParser.java @@ -116,7 +116,16 @@ public class TextQueryParser { } if (mapper != null && mapper.useFieldQueryWithQueryString()) { - return wrapSmartNameQuery(mapper.fieldQuery(text, parseContext), smartNameFieldMappers, parseContext); + if (smartNameFieldMappers.hasDocMapper()) { + String[] previousTypes = QueryParseContext.setTypesWithPrevious(new String[]{smartNameFieldMappers.docMapper().type()}); + try { + return wrapSmartNameQuery(mapper.fieldQuery(text, parseContext), smartNameFieldMappers, parseContext); + } finally { + QueryParseContext.setTypes(previousTypes); + } + } else { + return wrapSmartNameQuery(mapper.fieldQuery(text, parseContext), smartNameFieldMappers, parseContext); + } } Analyzer analyzer = null; diff --git a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java index 33d08874600..768281d77e5 100644 --- a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java +++ b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java @@ -52,6 +52,7 @@ import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.merge.scheduler.MergeSchedulerProvider; import org.elasticsearch.index.query.IndexQueryParserService; +import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.nested.IncludeAllChildrenQuery; import org.elasticsearch.index.search.nested.NonNestedDocsFilter; @@ -395,7 +396,12 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I if (querySourceLength == 0) { query = Queries.MATCH_ALL_QUERY; } else { - query = queryParserService.parse(querySource, querySourceOffset, querySourceLength).query(); + try { + QueryParseContext.setTypes(types); + query = queryParserService.parse(querySource, querySourceOffset, querySourceLength).query(); + } finally { + QueryParseContext.removeTypes(); + } } // wrap it in filter, cache it, and constant score it // Don't cache it, since it might be very different queries each time... From bb6fb6e08341d7c2156327ed80354ca827f19a1a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 28 Jan 2012 00:26:53 +0200 Subject: [PATCH 210/270] improve test to wait for 2 nodes --- .../test/integration/gateway/local/QuorumLocalGatewayTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/elasticsearch/test/integration/gateway/local/QuorumLocalGatewayTests.java b/src/test/java/org/elasticsearch/test/integration/gateway/local/QuorumLocalGatewayTests.java index ca568feb289..89c1fe314f5 100644 --- a/src/test/java/org/elasticsearch/test/integration/gateway/local/QuorumLocalGatewayTests.java +++ b/src/test/java/org/elasticsearch/test/integration/gateway/local/QuorumLocalGatewayTests.java @@ -144,7 +144,7 @@ public class QuorumLocalGatewayTests extends AbstractNodesTests { closeNode("node1"); logger.info("--> running cluster_health (wait for the shards to startup)"); - clusterHealth = client("node2").admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(4)).actionGet(); + clusterHealth = client("node2").admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForNodes("2").waitForActiveShards(4)).actionGet(); logger.info("--> done cluster_health, status " + clusterHealth.status()); assertThat(clusterHealth.timedOut(), equalTo(false)); assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.YELLOW)); From b2a333269834ecb4c9336452ef8f7086f1c70db2 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 29 Jan 2012 18:17:37 +0200 Subject: [PATCH 211/270] Terms Facet sorting does not handle null values properly with new JDK 7 sorting, closes #1647. --- .../datehistogram/DateHistogramFacet.java | 11 +++++-- .../facet/histogram/HistogramFacet.java | 11 +++++-- .../facet/termsstats/TermsStatsFacet.java | 33 +++++++++++++++++-- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacet.java b/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacet.java index c54c20dc84f..3f574d70171 100644 --- a/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacet.java @@ -27,8 +27,6 @@ import java.util.List; /** * A date histogram facet. - * - * */ public interface DateHistogramFacet extends Facet, Iterable { @@ -54,6 +52,9 @@ public interface DateHistogramFacet extends Facet, Iterable { @@ -54,6 +52,9 @@ public interface HistogramFacet extends Facet, Iterable { public int compare(Entry o1, Entry o2) { // push nulls to the end if (o1 == null) { + if (o2 == null) { + return 0; + } return 1; } if (o2 == null) { @@ -68,6 +69,9 @@ public interface HistogramFacet extends Facet, Iterable { public int compare(Entry o1, Entry o2) { // push nulls to the end if (o1 == null) { + if (o2 == null) { + return 0; + } return 1; } if (o2 == null) { @@ -82,6 +86,9 @@ public interface HistogramFacet extends Facet, Iterable { public int compare(Entry o1, Entry o2) { // push nulls to the end if (o1 == null) { + if (o2 == null) { + return 0; + } return 1; } if (o2 == null) { diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/TermsStatsFacet.java b/src/main/java/org/elasticsearch/search/facet/termsstats/TermsStatsFacet.java index ad171c5d6d1..5b8bcfb7346 100644 --- a/src/main/java/org/elasticsearch/search/facet/termsstats/TermsStatsFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/termsstats/TermsStatsFacet.java @@ -63,6 +63,9 @@ public interface TermsStatsFacet extends Facet, Iterable public int compare(Entry o1, Entry o2) { // push nulls to the end if (o1 == null) { + if (o2 == null) { + return 0; + } return 1; } if (o2 == null) { @@ -71,9 +74,6 @@ public interface TermsStatsFacet extends Facet, Iterable int i = (o2.count() < o1.count() ? -1 : (o1.count() == o2.count() ? 0 : 1)); if (i == 0) { i = o2.compareTo(o1); - if (i == 0) { - i = System.identityHashCode(o2) - System.identityHashCode(o1); - } } return i; } @@ -87,6 +87,9 @@ public interface TermsStatsFacet extends Facet, Iterable public int compare(Entry o1, Entry o2) { // push nulls to the end if (o1 == null) { + if (o2 == null) { + return 0; + } return 1; } if (o2 == null) { @@ -104,6 +107,9 @@ public interface TermsStatsFacet extends Facet, Iterable public int compare(Entry o1, Entry o2) { // push nulls to the end if (o1 == null) { + if (o2 == null) { + return 0; + } return 1; } if (o2 == null) { @@ -125,6 +131,9 @@ public interface TermsStatsFacet extends Facet, Iterable public int compare(Entry o1, Entry o2) { // push nulls to the end if (o1 == null) { + if (o2 == null) { + return 0; + } return 1; } if (o2 == null) { @@ -139,6 +148,9 @@ public interface TermsStatsFacet extends Facet, Iterable public int compare(Entry o1, Entry o2) { // push nulls to the end if (o1 == null) { + if (o2 == null) { + return 0; + } return 1; } if (o2 == null) { @@ -159,6 +171,9 @@ public interface TermsStatsFacet extends Facet, Iterable public int compare(Entry o1, Entry o2) { // push nulls to the end if (o1 == null) { + if (o2 == null) { + return 0; + } return 1; } if (o2 == null) { @@ -173,6 +188,9 @@ public interface TermsStatsFacet extends Facet, Iterable public int compare(Entry o1, Entry o2) { // push nulls to the end if (o1 == null) { + if (o2 == null) { + return 0; + } return 1; } if (o2 == null) { @@ -192,6 +210,9 @@ public interface TermsStatsFacet extends Facet, Iterable public int compare(Entry o1, Entry o2) { // push nulls to the end if (o1 == null) { + if (o2 == null) { + return 0; + } return 1; } if (o2 == null) { @@ -205,6 +226,9 @@ public interface TermsStatsFacet extends Facet, Iterable public int compare(Entry o1, Entry o2) { // push nulls to the end if (o1 == null) { + if (o2 == null) { + return 0; + } return 1; } if (o2 == null) { @@ -224,6 +248,9 @@ public interface TermsStatsFacet extends Facet, Iterable public int compare(Entry o1, Entry o2) { // push nulls to the end if (o1 == null) { + if (o2 == null) { + return 0; + } return 1; } if (o2 == null) { From 49b6d70dfdeeac2a9a12647a83b9ef3613346faa Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 29 Jan 2012 21:09:11 +0200 Subject: [PATCH 212/270] Query DSL: prefix query to support _id, closes #1648. --- .../lucene/queryParser/MapperQueryParser.java | 22 +++++++-- .../index/mapper/FieldMapper.java | 5 ++ .../mapper/core/AbstractFieldMapper.java | 14 ++++++ .../index/mapper/internal/IdFieldMapper.java | 46 +++++++++++++++++-- .../index/query/ExistsFilterParser.java | 6 +-- .../index/query/MissingFilterParser.java | 6 +-- .../index/query/PrefixFilterParser.java | 14 ++++-- .../index/query/PrefixQueryParser.java | 24 +++++++--- .../index/query/TermFilterParser.java | 18 ++++---- .../index/query/TermQueryParser.java | 18 ++++---- .../index/query/TermsQueryParser.java | 36 +++++++++------ .../index/query/WildcardQueryParser.java | 8 ++-- .../index/search/TextQueryParser.java | 10 ++-- .../search/simple/SimpleSearchTests.java | 13 ++++-- 14 files changed, 165 insertions(+), 75 deletions(-) diff --git a/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java b/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java index 4a5d98c4caa..6c3a51f45cd 100644 --- a/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java +++ b/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java @@ -190,7 +190,6 @@ public class MapperQueryParser extends QueryParser { @Override protected Query getPrefixQuery(String field, String termStr) throws ParseException { - String indexedNameField = field; currentMapper = null; Analyzer oldAnalyzer = analyzer; try { @@ -201,11 +200,26 @@ public class MapperQueryParser extends QueryParser { } currentMapper = fieldMappers.fieldMappers().mapper(); if (currentMapper != null) { - indexedNameField = currentMapper.names().indexName(); + Query query = null; + if (currentMapper.useFieldQueryWithQueryString()) { + if (fieldMappers.hasDocMapper()) { + String[] previousTypes = QueryParseContext.setTypesWithPrevious(new String[]{fieldMappers.docMapper().type()}); + try { + query = currentMapper.prefixQuery(termStr, multiTermRewriteMethod, parseContext); + } finally { + QueryParseContext.setTypes(previousTypes); + } + } else { + query = currentMapper.prefixQuery(termStr, multiTermRewriteMethod, parseContext); + } + } + if (query == null) { + query = super.getPrefixQuery(currentMapper.names().indexName(), termStr); + } + return wrapSmartNameQuery(query, fieldMappers, parseContext); } - return wrapSmartNameQuery(getPossiblyAnalyzedPrefixQuery(indexedNameField, termStr), fieldMappers, parseContext); } - return getPossiblyAnalyzedPrefixQuery(indexedNameField, termStr); + return super.getPrefixQuery(field, termStr); } finally { analyzer = oldAnalyzer; } diff --git a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 38a1f29007b..ddec7dc6efe 100644 --- a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -24,6 +24,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.Term; import org.apache.lucene.search.Filter; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.field.data.FieldDataType; @@ -173,6 +174,10 @@ public interface FieldMapper { Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions); + Query prefixQuery(String value, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryParseContext context); + + Filter prefixFilter(String value, @Nullable QueryParseContext context); + /** * A term query to use when parsing a query string. Can return null. */ diff --git a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java index 37bec35b5e8..f2771a30b28 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java @@ -380,6 +380,20 @@ public abstract class AbstractFieldMapper implements FieldMapper, Mapper { return new FuzzyQuery(names().createIndexNameTerm(value), (float) minSim, prefixLength, maxExpansions); } + @Override + public Query prefixQuery(String value, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryParseContext context) { + PrefixQuery query = new PrefixQuery(names().createIndexNameTerm(indexedValue(value))); + if (method != null) { + query.setRewriteMethod(method); + } + return query; + } + + @Override + public Filter prefixFilter(String value, @Nullable QueryParseContext context) { + return new PrefixFilter(names().createIndexNameTerm(indexedValue(value))); + } + @Override public Query rangeQuery(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { return new TermRangeQuery(names.indexName(), diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java index 60b7f4b9bc6..e7e6c0a6933 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java @@ -20,15 +20,15 @@ package org.elasticsearch.index.mapper.internal; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Fieldable; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.Query; +import org.apache.lucene.search.*; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.lucene.search.XBooleanFilter; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.*; @@ -37,6 +37,7 @@ import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.search.UidFilter; import java.io.IOException; +import java.util.Collection; import java.util.Map; import static org.elasticsearch.index.mapper.MapperBuilders.id; @@ -175,6 +176,45 @@ public class IdFieldMapper extends AbstractFieldMapper implements Intern return new UidFilter(context.queryTypes(), ImmutableList.of(value), context.indexCache().bloomCache()); } + @Override + public Query prefixQuery(String value, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryParseContext context) { + if (indexed() || context == null) { + return super.prefixQuery(value, method, context); + } + Collection queryTypes = context.queryTypes(); + if (queryTypes.size() == 1) { + PrefixQuery prefixQuery = new PrefixQuery(UidFieldMapper.TERM_FACTORY.createTerm(Uid.createUid(Iterables.getFirst(queryTypes, null), value))); + if (method != null) { + prefixQuery.setRewriteMethod(method); + } + } + BooleanQuery query = new BooleanQuery(); + for (String queryType : queryTypes) { + PrefixQuery prefixQuery = new PrefixQuery(UidFieldMapper.TERM_FACTORY.createTerm(Uid.createUid(queryType, value))); + if (method != null) { + prefixQuery.setRewriteMethod(method); + } + query.add(prefixQuery, BooleanClause.Occur.SHOULD); + } + return query; + } + + @Override + public Filter prefixFilter(String value, @Nullable QueryParseContext context) { + if (indexed() || context == null) { + return super.prefixFilter(value, context); + } + Collection queryTypes = context.queryTypes(); + if (queryTypes.size() == 1) { + return new PrefixFilter(UidFieldMapper.TERM_FACTORY.createTerm(Uid.createUid(Iterables.getFirst(queryTypes, null), value))); + } + XBooleanFilter filter = new XBooleanFilter(); + for (String queryType : queryTypes) { + filter.addShould(new PrefixFilter(UidFieldMapper.TERM_FACTORY.createTerm(Uid.createUid(queryType, value)))); + } + return filter; + } + @Override public void preParse(ParseContext context) throws IOException { if (context.sourceToParse().id() != null) { diff --git a/src/main/java/org/elasticsearch/index/query/ExistsFilterParser.java b/src/main/java/org/elasticsearch/index/query/ExistsFilterParser.java index c6ef6825d5e..a2f1fe4305c 100644 --- a/src/main/java/org/elasticsearch/index/query/ExistsFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/ExistsFilterParser.java @@ -74,10 +74,8 @@ public class ExistsFilterParser implements FilterParser { Filter filter = null; MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); - if (smartNameFieldMappers != null) { - if (smartNameFieldMappers.hasMapper()) { - filter = smartNameFieldMappers.mapper().rangeFilter(null, null, true, true); - } + if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { + filter = smartNameFieldMappers.mapper().rangeFilter(null, null, true, true); } if (filter == null) { filter = new TermRangeFilter(fieldName, null, null, true, true); diff --git a/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java b/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java index 17978ce5f67..9495f0be6cf 100644 --- a/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java @@ -75,10 +75,8 @@ public class MissingFilterParser implements FilterParser { Filter filter = null; MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); - if (smartNameFieldMappers != null) { - if (smartNameFieldMappers.hasMapper()) { - filter = smartNameFieldMappers.mapper().rangeFilter(null, null, true, true); - } + if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { + filter = smartNameFieldMappers.mapper().rangeFilter(null, null, true, true); } if (filter == null) { filter = new TermRangeFilter(fieldName, null, null, true, true); diff --git a/src/main/java/org/elasticsearch/index/query/PrefixFilterParser.java b/src/main/java/org/elasticsearch/index/query/PrefixFilterParser.java index 41492a0b197..5d2ad770d8b 100644 --- a/src/main/java/org/elasticsearch/index/query/PrefixFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/PrefixFilterParser.java @@ -83,10 +83,16 @@ public class PrefixFilterParser implements FilterParser { Filter filter = null; MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); - if (smartNameFieldMappers != null) { - if (smartNameFieldMappers.hasMapper()) { - value = smartNameFieldMappers.mapper().indexedValue(value); - filter = new PrefixFilter(smartNameFieldMappers.mapper().names().createIndexNameTerm(value)); + if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { + if (smartNameFieldMappers.hasDocMapper()) { + String[] previousTypes = QueryParseContext.setTypesWithPrevious(new String[]{smartNameFieldMappers.docMapper().type()}); + try { + filter = smartNameFieldMappers.mapper().prefixFilter(value, parseContext); + } finally { + QueryParseContext.setTypes(previousTypes); + } + } else { + filter = smartNameFieldMappers.mapper().prefixFilter(value, parseContext); } } if (filter == null) { diff --git a/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java b/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java index ef17de274bb..07d780a4b60 100644 --- a/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.Term; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.inject.Inject; @@ -90,18 +91,27 @@ public class PrefixQueryParser implements QueryParser { throw new QueryParsingException(parseContext.index(), "No value specified for prefix query"); } - PrefixQuery query = null; + MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(rewriteMethod); + + Query query = null; MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); - if (smartNameFieldMappers != null) { - if (smartNameFieldMappers.hasMapper()) { - value = smartNameFieldMappers.mapper().indexedValue(value); - query = new PrefixQuery(smartNameFieldMappers.mapper().names().createIndexNameTerm(value)); + if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { + if (smartNameFieldMappers.hasDocMapper()) { + String[] previousTypes = QueryParseContext.setTypesWithPrevious(new String[]{smartNameFieldMappers.docMapper().type()}); + try { + query = smartNameFieldMappers.mapper().prefixQuery(value, method, parseContext); + } finally { + QueryParseContext.setTypes(previousTypes); + } + } else { + query = smartNameFieldMappers.mapper().prefixQuery(value, method, parseContext); } } if (query == null) { - query = new PrefixQuery(new Term(fieldName, value)); + PrefixQuery prefixQuery = new PrefixQuery(new Term(fieldName, value)); + prefixQuery.setRewriteMethod(method); + query = prefixQuery; } - query.setRewriteMethod(QueryParsers.parseRewriteMethod(rewriteMethod)); query.setBoost(boost); return wrapSmartNameQuery(query, smartNameFieldMappers, parseContext); } diff --git a/src/main/java/org/elasticsearch/index/query/TermFilterParser.java b/src/main/java/org/elasticsearch/index/query/TermFilterParser.java index 7c3257bd69e..2e0bf11b2ec 100644 --- a/src/main/java/org/elasticsearch/index/query/TermFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/TermFilterParser.java @@ -86,18 +86,16 @@ public class TermFilterParser implements FilterParser { Filter filter = null; MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); - if (smartNameFieldMappers != null) { - if (smartNameFieldMappers.hasMapper()) { - if (smartNameFieldMappers.hasDocMapper()) { - String[] previousTypes = QueryParseContext.setTypesWithPrevious(new String[]{smartNameFieldMappers.docMapper().type()}); - try { - filter = smartNameFieldMappers.mapper().fieldFilter(value, parseContext); - } finally { - QueryParseContext.setTypes(previousTypes); - } - } else { + if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { + if (smartNameFieldMappers.hasDocMapper()) { + String[] previousTypes = QueryParseContext.setTypesWithPrevious(new String[]{smartNameFieldMappers.docMapper().type()}); + try { filter = smartNameFieldMappers.mapper().fieldFilter(value, parseContext); + } finally { + QueryParseContext.setTypes(previousTypes); } + } else { + filter = smartNameFieldMappers.mapper().fieldFilter(value, parseContext); } } if (filter == null) { diff --git a/src/main/java/org/elasticsearch/index/query/TermQueryParser.java b/src/main/java/org/elasticsearch/index/query/TermQueryParser.java index 8de9d6df915..6eddfeecb64 100644 --- a/src/main/java/org/elasticsearch/index/query/TermQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/TermQueryParser.java @@ -89,18 +89,16 @@ public class TermQueryParser implements QueryParser { Query query = null; MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); - if (smartNameFieldMappers != null) { - if (smartNameFieldMappers.hasMapper()) { - if (smartNameFieldMappers.hasDocMapper()) { - String[] previousTypes = QueryParseContext.setTypesWithPrevious(new String[]{smartNameFieldMappers.docMapper().type()}); - try { - query = smartNameFieldMappers.mapper().fieldQuery(value, parseContext); - } finally { - QueryParseContext.setTypes(previousTypes); - } - } else { + if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { + if (smartNameFieldMappers.hasDocMapper()) { + String[] previousTypes = QueryParseContext.setTypesWithPrevious(new String[]{smartNameFieldMappers.docMapper().type()}); + try { query = smartNameFieldMappers.mapper().fieldQuery(value, parseContext); + } finally { + QueryParseContext.setTypes(previousTypes); } + } else { + query = smartNameFieldMappers.mapper().fieldQuery(value, parseContext); } } if (query == null) { diff --git a/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java b/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java index 721a919fd00..cf4dc310de8 100644 --- a/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java @@ -97,25 +97,33 @@ public class TermsQueryParser implements QueryParser { FieldMapper mapper = null; MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); - if (smartNameFieldMappers != null) { - if (smartNameFieldMappers.hasMapper()) { - mapper = smartNameFieldMappers.mapper(); + String[] previousTypes = null; + if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { + mapper = smartNameFieldMappers.mapper(); + if (smartNameFieldMappers.hasDocMapper()) { + previousTypes = QueryParseContext.setTypesWithPrevious(new String[]{smartNameFieldMappers.docMapper().type()}); } } - BooleanQuery query = new BooleanQuery(disableCoord); - for (String value : values) { - if (mapper != null) { - query.add(new BooleanClause(mapper.fieldQuery(value, parseContext), BooleanClause.Occur.SHOULD)); - } else { - query.add(new TermQuery(new Term(fieldName, value)), BooleanClause.Occur.SHOULD); + try { + BooleanQuery query = new BooleanQuery(disableCoord); + for (String value : values) { + if (mapper != null) { + query.add(new BooleanClause(mapper.fieldQuery(value, parseContext), BooleanClause.Occur.SHOULD)); + } else { + query.add(new TermQuery(new Term(fieldName, value)), BooleanClause.Occur.SHOULD); + } + } + query.setBoost(boost); + if (minimumNumberShouldMatch != -1) { + query.setMinimumNumberShouldMatch(minimumNumberShouldMatch); + } + return wrapSmartNameQuery(optimizeQuery(fixNegativeQueryIfNeeded(query)), smartNameFieldMappers, parseContext); + } finally { + if (smartNameFieldMappers != null && smartNameFieldMappers.hasDocMapper()) { + QueryParseContext.setTypes(previousTypes); } } - query.setBoost(boost); - if (minimumNumberShouldMatch != -1) { - query.setMinimumNumberShouldMatch(minimumNumberShouldMatch); - } - return wrapSmartNameQuery(optimizeQuery(fixNegativeQueryIfNeeded(query)), smartNameFieldMappers, parseContext); } } diff --git a/src/main/java/org/elasticsearch/index/query/WildcardQueryParser.java b/src/main/java/org/elasticsearch/index/query/WildcardQueryParser.java index bdad45a6161..df54824194a 100644 --- a/src/main/java/org/elasticsearch/index/query/WildcardQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/WildcardQueryParser.java @@ -91,11 +91,9 @@ public class WildcardQueryParser implements QueryParser { } MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); - if (smartNameFieldMappers != null) { - if (smartNameFieldMappers.hasMapper()) { - fieldName = smartNameFieldMappers.mapper().names().indexName(); - value = smartNameFieldMappers.mapper().indexedValue(value); - } + if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { + fieldName = smartNameFieldMappers.mapper().names().indexName(); + value = smartNameFieldMappers.mapper().indexedValue(value); } WildcardQuery query = new WildcardQuery(new Term(fieldName, value)); diff --git a/src/main/java/org/elasticsearch/index/search/TextQueryParser.java b/src/main/java/org/elasticsearch/index/search/TextQueryParser.java index 4ecdd59bd1e..c0a976da726 100644 --- a/src/main/java/org/elasticsearch/index/search/TextQueryParser.java +++ b/src/main/java/org/elasticsearch/index/search/TextQueryParser.java @@ -106,13 +106,9 @@ public class TextQueryParser { FieldMapper mapper = null; String field = fieldName; MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); - if (smartNameFieldMappers != null) { - if (smartNameFieldMappers.hasMapper()) { - mapper = smartNameFieldMappers.mapper(); - if (mapper != null) { - field = mapper.names().indexName(); - } - } + if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { + mapper = smartNameFieldMappers.mapper(); + field = mapper.names().indexName(); } if (mapper != null && mapper.useFieldQueryWithQueryString()) { diff --git a/src/test/java/org/elasticsearch/test/integration/search/simple/SimpleSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/simple/SimpleSearchTests.java index 4fea1082220..749735db270 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/simple/SimpleSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/simple/SimpleSearchTests.java @@ -81,12 +81,19 @@ public class SimpleSearchTests extends AbstractNodesTests { client.admin().indices().prepareDelete().execute().actionGet(); client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 1)).execute().actionGet(); - client.prepareIndex("test", "type", "1").setSource("field", "value").setRefresh(true).execute().actionGet(); + client.prepareIndex("test", "type", "XXX1").setSource("field", "value").setRefresh(true).execute().actionGet(); // id is not indexed, but lets see that we automatically convert to - SearchResponse searchResponse = client.prepareSearch().setQuery(QueryBuilders.termQuery("_id", "1")).execute().actionGet(); + SearchResponse searchResponse = client.prepareSearch().setQuery(QueryBuilders.termQuery("_id", "XXX1")).execute().actionGet(); assertThat(searchResponse.hits().totalHits(), equalTo(1l)); - searchResponse = client.prepareSearch().setQuery(QueryBuilders.queryString("_id:1")).execute().actionGet(); + searchResponse = client.prepareSearch().setQuery(QueryBuilders.queryString("_id:XXX1")).execute().actionGet(); + assertThat(searchResponse.hits().totalHits(), equalTo(1l)); + + // id is not index, but we can automatically support prefix as well + searchResponse = client.prepareSearch().setQuery(QueryBuilders.prefixQuery("_id", "XXX")).execute().actionGet(); + assertThat(searchResponse.hits().totalHits(), equalTo(1l)); + + searchResponse = client.prepareSearch().setQuery(QueryBuilders.queryString("_id:XXX*")).execute().actionGet(); assertThat(searchResponse.hits().totalHits(), equalTo(1l)); } From 25763e2124643bddad5c148a07e735cac08c7554 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 29 Jan 2012 22:05:16 +0200 Subject: [PATCH 213/270] Query DSL: prefix query to support _id, closes #1648. (fix support analysis for prefix is configured) --- .../java/org/apache/lucene/queryParser/MapperQueryParser.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java b/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java index 6c3a51f45cd..ad69d854e90 100644 --- a/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java +++ b/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java @@ -214,12 +214,12 @@ public class MapperQueryParser extends QueryParser { } } if (query == null) { - query = super.getPrefixQuery(currentMapper.names().indexName(), termStr); + query = getPossiblyAnalyzedPrefixQuery(currentMapper.names().indexName(), termStr); } return wrapSmartNameQuery(query, fieldMappers, parseContext); } } - return super.getPrefixQuery(field, termStr); + return getPossiblyAnalyzedPrefixQuery(field, termStr); } finally { analyzer = oldAnalyzer; } From 8e6b171205760044db06a8d6bc904037cd42b2c6 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 29 Jan 2012 22:52:52 +0200 Subject: [PATCH 214/270] remove test sysout --- .../java/org/elasticsearch/index/query/NestedQueryParser.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java b/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java index e8fb0b17d3a..899b7436e3b 100644 --- a/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java @@ -126,9 +126,6 @@ public class NestedQueryParser implements QueryParser { Filter childFilter = parseContext.cacheFilter(objectMapper.nestedTypeFilter(), null); usAsParentFilter.filter = childFilter; - if (usAsParentFilter.filter == null) { - System.out.println("HELLO"); - } // wrap the child query to only work on the nested path type query = new FilteredQuery(query, childFilter); From 70c334ec01afa90ee92ee8d8c6571aaf6757ff7b Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 30 Jan 2012 01:43:18 +0200 Subject: [PATCH 215/270] Index Allocation: allow to specify maximum total number of shards per node, closes #1650. --- .../decider/AllocationDeciders.java | 1 + .../decider/AllocationDecidersModule.java | 6 + .../decider/ShardsLimitAllocationDecider.java | 103 +++++++++ .../ShardsLimitAllocationTests.java | 202 ++++++++++++++++++ 4 files changed, 312 insertions(+) create mode 100644 src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java create mode 100644 src/test/java/org/elasticsearch/test/unit/cluster/routing/allocation/ShardsLimitAllocationTests.java diff --git a/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java b/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java index c1a8e561241..72a7b6f8b31 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java +++ b/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java @@ -47,6 +47,7 @@ public class AllocationDeciders extends AllocationDecider { .add(new ConcurrentRebalanceAllocationDecider(settings, nodeSettingsService)) .add(new DisableAllocationDecider(settings, nodeSettingsService)) .add(new AwarenessAllocationDecider(settings, nodeSettingsService)) + .add(new ShardsLimitAllocationDecider(settings)) .build() ); } diff --git a/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersModule.java b/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersModule.java index 9abc94410dc..30a69977b13 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersModule.java +++ b/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersModule.java @@ -38,6 +38,11 @@ public class AllocationDecidersModule extends AbstractModule { this.settings = settings; } + public AllocationDecidersModule add(Class allocationDecider) { + this.allocations.add(allocationDecider); + return this; + } + @Override protected void configure() { Multibinder allocationMultibinder = Multibinder.newSetBinder(binder(), AllocationDecider.class); @@ -50,6 +55,7 @@ public class AllocationDecidersModule extends AbstractModule { allocationMultibinder.addBinding().to(ConcurrentRebalanceAllocationDecider.class); allocationMultibinder.addBinding().to(DisableAllocationDecider.class); allocationMultibinder.addBinding().to(AwarenessAllocationDecider.class); + allocationMultibinder.addBinding().to(ShardsLimitAllocationDecider.class); for (Class allocation : allocations) { allocationMultibinder.addBinding().to(allocation); } diff --git a/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java b/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java new file mode 100644 index 00000000000..61c2cb4f319 --- /dev/null +++ b/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java @@ -0,0 +1,103 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.routing.allocation.decider; + +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.MutableShardRouting; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; + +import java.util.List; + +/** + * + */ +public class ShardsLimitAllocationDecider extends AllocationDecider { + + public static final String INDEX_TOTAL_SHARDS_PER_NODE = "index.routing.allocation.total_shards_per_node"; + + static { + IndexMetaData.addDynamicSettings( + INDEX_TOTAL_SHARDS_PER_NODE + ); + } + + @Inject + public ShardsLimitAllocationDecider(Settings settings) { + super(settings); + } + + @Override + public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + IndexMetaData indexMd = allocation.routingNodes().metaData().index(shardRouting.index()); + int totalShardsPerNode = indexMd.settings().getAsInt(INDEX_TOTAL_SHARDS_PER_NODE, -1); + if (totalShardsPerNode <= 0) { + return Decision.YES; + } + + int nodeCount = 0; + List shards = node.shards(); + for (int i = 0; i < shards.size(); i++) { + MutableShardRouting nodeShard = shards.get(i); + if (!nodeShard.index().equals(shardRouting.index())) { + continue; + } + // don't count relocating shards... + if (nodeShard.relocating()) { + continue; + } + nodeCount++; + } + if (nodeCount >= totalShardsPerNode) { + return Decision.NO; + } + return Decision.YES; + } + + @Override + public boolean canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + IndexMetaData indexMd = allocation.routingNodes().metaData().index(shardRouting.index()); + int totalShardsPerNode = indexMd.settings().getAsInt(INDEX_TOTAL_SHARDS_PER_NODE, -1); + if (totalShardsPerNode <= 0) { + return true; + } + + int nodeCount = 0; + List shards = node.shards(); + for (int i = 0; i < shards.size(); i++) { + MutableShardRouting nodeShard = shards.get(i); + if (!nodeShard.index().equals(shardRouting.index())) { + continue; + } + // don't count relocating shards... + if (nodeShard.relocating()) { + continue; + } + nodeCount++; + } + if (nodeCount > totalShardsPerNode) { + return false; + } + return true; + } +} diff --git a/src/test/java/org/elasticsearch/test/unit/cluster/routing/allocation/ShardsLimitAllocationTests.java b/src/test/java/org/elasticsearch/test/unit/cluster/routing/allocation/ShardsLimitAllocationTests.java new file mode 100644 index 00000000000..47e2a2f7200 --- /dev/null +++ b/src/test/java/org/elasticsearch/test/unit/cluster/routing/allocation/ShardsLimitAllocationTests.java @@ -0,0 +1,202 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.unit.cluster.routing.allocation; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.routing.MutableShardRouting; +import org.elasticsearch.cluster.routing.RoutingNodes; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.testng.annotations.Test; + +import static org.elasticsearch.cluster.ClusterState.newClusterStateBuilder; +import static org.elasticsearch.cluster.metadata.IndexMetaData.newIndexMetaDataBuilder; +import static org.elasticsearch.cluster.metadata.MetaData.newMetaDataBuilder; +import static org.elasticsearch.cluster.node.DiscoveryNodes.newNodesBuilder; +import static org.elasticsearch.cluster.routing.RoutingBuilders.indexRoutingTable; +import static org.elasticsearch.cluster.routing.RoutingBuilders.routingTable; +import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; +import static org.elasticsearch.test.unit.cluster.routing.allocation.RoutingAllocationTests.newNode; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +/** + */ +@Test +public class ShardsLimitAllocationTests { + + private final ESLogger logger = Loggers.getLogger(ShardsLimitAllocationTests.class); + + @Test + public void indexLevelShardsLimitAllocate() { + AllocationService strategy = new AllocationService(settingsBuilder().put("cluster.routing.allocation.concurrent_recoveries", 10).build()); + + logger.info("Building initial routing table"); + + MetaData metaData = newMetaDataBuilder() + .put(newIndexMetaDataBuilder("test").settings(ImmutableSettings.settingsBuilder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE, 2))) + .build(); + + RoutingTable routingTable = routingTable() + .add(indexRoutingTable("test").initializeEmpty(metaData.index("test"))) + .build(); + + ClusterState clusterState = newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build(); + logger.info("Adding two nodes and performing rerouting"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().put(newNode("node1")).put(newNode("node2"))).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.readOnlyRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(2)); + assertThat(clusterState.readOnlyRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(2)); + + logger.info("Start the primary shards"); + RoutingNodes routingNodes = clusterState.routingNodes(); + routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.readOnlyRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(2)); + assertThat(clusterState.readOnlyRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(0)); + assertThat(clusterState.readOnlyRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(2)); + assertThat(clusterState.readOnlyRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(0)); + assertThat(clusterState.readOnlyRoutingNodes().unassigned().size(), equalTo(4)); + + logger.info("Do another reroute, make sure its still not allocated"); + routingNodes = clusterState.routingNodes(); + routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + } + + @Test + public void indexLevelShardsLimitRemain() { + AllocationService strategy = new AllocationService(settingsBuilder() + .put("cluster.routing.allocation.concurrent_recoveries", 10) + .put("cluster.routing.allocation.node_initial_primaries_recoveries", 10) + .put("cluster.routing.allocation.allow_rebalance", "always") + .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1) + .build()); + + logger.info("Building initial routing table"); + + MetaData metaData = newMetaDataBuilder() + .put(newIndexMetaDataBuilder("test").settings(ImmutableSettings.settingsBuilder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + )) + .build(); + + RoutingTable routingTable = routingTable() + .add(indexRoutingTable("test").initializeEmpty(metaData.index("test"))) + .build(); + + ClusterState clusterState = newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build(); + logger.info("Adding one node and reroute"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().put(newNode("node1"))).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + logger.info("Start the primary shards"); + RoutingNodes routingNodes = clusterState.routingNodes(); + routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.readOnlyRoutingNodes().numberOfShardsOfType(STARTED), equalTo(5)); + + logger.info("add another index with 5 shards"); + metaData = newMetaDataBuilder().metaData(metaData) + .put(newIndexMetaDataBuilder("test1").settings(ImmutableSettings.settingsBuilder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + )) + .build(); + routingTable = routingTable().routingTable(routingTable) + .add(indexRoutingTable("test1").initializeEmpty(metaData.index("test1"))) + .build(); + + clusterState = newClusterStateBuilder().state(clusterState).metaData(metaData).routingTable(routingTable).build(); + + logger.info("Add another one node and reroute"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()).put(newNode("node2"))).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + routingNodes = clusterState.routingNodes(); + routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.readOnlyRoutingNodes().numberOfShardsOfType(STARTED), equalTo(10)); + + for (MutableShardRouting shardRouting : clusterState.readOnlyRoutingNodes().node("node1")) { + assertThat(shardRouting.index(), equalTo("test")); + } + for (MutableShardRouting shardRouting : clusterState.readOnlyRoutingNodes().node("node2")) { + assertThat(shardRouting.index(), equalTo("test1")); + } + + logger.info("update " + ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE + " for test, see that things move"); + metaData = newMetaDataBuilder().metaData(metaData) + .put(newIndexMetaDataBuilder("test").settings(ImmutableSettings.settingsBuilder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE, 3) + )) + .build(); + + + clusterState = newClusterStateBuilder().state(clusterState).metaData(metaData).build(); + + logger.info("reroute after setting"); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.readOnlyRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(3)); + assertThat(clusterState.readOnlyRoutingNodes().node("node1").numberOfShardsWithState(RELOCATING), equalTo(2)); + + logger.info("start the moving shards, a shard from test1 should move back to node1"); + routingNodes = clusterState.routingNodes(); + routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.readOnlyRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(3)); + assertThat(clusterState.readOnlyRoutingNodes().node("node1").numberOfShardsWithState(RELOCATING), equalTo(0)); + assertThat(clusterState.readOnlyRoutingNodes().node("node1").numberOfShardsWithState(INITIALIZING), equalTo(2)); + assertThat(clusterState.readOnlyRoutingNodes().node("node2").numberOfShardsWithState(STARTED), equalTo(5)); + assertThat(clusterState.readOnlyRoutingNodes().node("node2").numberOfShardsWithState(RELOCATING), equalTo(2)); + + logger.info("finalize movement, another shard will move"); + routingNodes = clusterState.routingNodes(); + routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.readOnlyRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(5)); + assertThat(clusterState.readOnlyRoutingNodes().node("node2").numberOfShardsWithState(STARTED), equalTo(5)); + } +} From f6deb4597094ac46ad862f1c5817db66c40c6dea Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 30 Jan 2012 01:58:51 +0200 Subject: [PATCH 216/270] Cluster Allocation: cluster.routing.allocation.allow_rebalance does not allow for rebalancing on relocating shard, closes #1651. --- .../allocation/decider/ClusterRebalanceAllocationDecider.java | 4 ++-- .../routing/allocation/ConcurrentRebalanceRoutingTests.java | 3 +-- .../routing/allocation/ShardsLimitAllocationTests.java | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java b/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java index c6ae0530911..98d4a3e39c3 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java +++ b/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java @@ -67,7 +67,7 @@ public class ClusterRebalanceAllocationDecider extends AllocationDecider { List shards = node.shards(); for (int i = 0; i < shards.size(); i++) { MutableShardRouting shard = shards.get(i); - if (shard.primary() && !shard.active()) { + if (shard.primary() && !shard.active() && shard.relocatingNodeId() == null) { return false; } } @@ -82,7 +82,7 @@ public class ClusterRebalanceAllocationDecider extends AllocationDecider { List shards = node.shards(); for (int i = 0; i < shards.size(); i++) { MutableShardRouting shard = shards.get(i); - if (!shard.active()) { + if (!shard.active() && shard.relocatingNodeId() == null) { return false; } } diff --git a/src/test/java/org/elasticsearch/test/unit/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java b/src/test/java/org/elasticsearch/test/unit/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java index 82a94298264..e0110e744a7 100644 --- a/src/test/java/org/elasticsearch/test/unit/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java +++ b/src/test/java/org/elasticsearch/test/unit/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java @@ -35,8 +35,8 @@ import static org.elasticsearch.cluster.node.DiscoveryNodes.newNodesBuilder; import static org.elasticsearch.cluster.routing.RoutingBuilders.indexRoutingTable; import static org.elasticsearch.cluster.routing.RoutingBuilders.routingTable; import static org.elasticsearch.cluster.routing.ShardRoutingState.*; -import static org.elasticsearch.test.unit.cluster.routing.allocation.RoutingAllocationTests.newNode; import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; +import static org.elasticsearch.test.unit.cluster.routing.allocation.RoutingAllocationTests.newNode; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -50,7 +50,6 @@ public class ConcurrentRebalanceRoutingTests { public void testClusterConcurrentRebalance() { AllocationService strategy = new AllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) - .put("cluster.routing.allocation.allow_rebalance", "always") .put("cluster.routing.allocation.cluster_concurrent_rebalance", 3) .build()); diff --git a/src/test/java/org/elasticsearch/test/unit/cluster/routing/allocation/ShardsLimitAllocationTests.java b/src/test/java/org/elasticsearch/test/unit/cluster/routing/allocation/ShardsLimitAllocationTests.java index 47e2a2f7200..aca39ea2ca3 100644 --- a/src/test/java/org/elasticsearch/test/unit/cluster/routing/allocation/ShardsLimitAllocationTests.java +++ b/src/test/java/org/elasticsearch/test/unit/cluster/routing/allocation/ShardsLimitAllocationTests.java @@ -100,7 +100,6 @@ public class ShardsLimitAllocationTests { AllocationService strategy = new AllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put("cluster.routing.allocation.node_initial_primaries_recoveries", 10) - .put("cluster.routing.allocation.allow_rebalance", "always") .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1) .build()); From 06da379f5045e0c1d7436a7757400f9ba5b7f993 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 30 Jan 2012 02:19:31 +0200 Subject: [PATCH 217/270] simplify check if there is explicit type name and doc mapper in the field to be used --- .../java/org/apache/lucene/queryParser/MapperQueryParser.java | 4 ++-- .../java/org/elasticsearch/index/mapper/MapperService.java | 4 ++++ .../org/elasticsearch/index/query/PrefixFilterParser.java | 2 +- .../java/org/elasticsearch/index/query/PrefixQueryParser.java | 2 +- .../java/org/elasticsearch/index/query/TermFilterParser.java | 2 +- .../java/org/elasticsearch/index/query/TermQueryParser.java | 2 +- .../java/org/elasticsearch/index/query/TermsFilterParser.java | 4 ++-- .../java/org/elasticsearch/index/query/TermsQueryParser.java | 4 ++-- .../org/elasticsearch/index/query/support/QueryParsers.java | 4 ++-- .../java/org/elasticsearch/index/search/TextQueryParser.java | 2 +- .../facet/datehistogram/CountDateHistogramFacetCollector.java | 2 +- .../facet/datehistogram/ValueDateHistogramFacetCollector.java | 2 +- .../datehistogram/ValueScriptDateHistogramFacetCollector.java | 2 +- .../search/facet/geodistance/GeoDistanceFacetCollector.java | 2 +- .../bounded/BoundedCountHistogramFacetCollector.java | 2 +- .../bounded/BoundedValueHistogramFacetCollector.java | 2 +- .../bounded/BoundedValueScriptHistogramFacetCollector.java | 2 +- .../histogram/unbounded/CountHistogramFacetCollector.java | 4 +--- .../histogram/unbounded/FullHistogramFacetCollector.java | 2 +- .../histogram/unbounded/ValueHistogramFacetCollector.java | 2 +- .../unbounded/ValueScriptHistogramFacetCollector.java | 2 +- .../search/facet/range/KeyValueRangeFacetCollector.java | 2 +- .../elasticsearch/search/facet/range/RangeFacetCollector.java | 2 +- .../search/facet/statistical/StatisticalFacetCollector.java | 2 +- .../search/facet/terms/bytes/TermsByteFacetCollector.java | 2 +- .../facet/terms/bytes/TermsByteOrdinalsFacetCollector.java | 2 +- .../search/facet/terms/doubles/TermsDoubleFacetCollector.java | 2 +- .../terms/doubles/TermsDoubleOrdinalsFacetCollector.java | 2 +- .../search/facet/terms/floats/TermsFloatFacetCollector.java | 2 +- .../facet/terms/floats/TermsFloatOrdinalsFacetCollector.java | 2 +- .../search/facet/terms/ints/TermsIntFacetCollector.java | 2 +- .../facet/terms/ints/TermsIntOrdinalsFacetCollector.java | 2 +- .../search/facet/terms/ip/TermsIpFacetCollector.java | 2 +- .../search/facet/terms/ip/TermsIpOrdinalsFacetCollector.java | 2 +- .../search/facet/terms/longs/TermsLongFacetCollector.java | 2 +- .../facet/terms/longs/TermsLongOrdinalsFacetCollector.java | 2 +- .../search/facet/terms/shorts/TermsShortFacetCollector.java | 2 +- .../facet/terms/shorts/TermsShortOrdinalsFacetCollector.java | 2 +- .../terms/strings/TermsStringOrdinalsFacetCollector.java | 2 +- .../termsstats/doubles/TermsStatsDoubleFacetCollector.java | 2 +- .../facet/termsstats/longs/TermsStatsLongFacetCollector.java | 2 +- .../termsstats/strings/TermsStatsStringFacetCollector.java | 2 +- 42 files changed, 49 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java b/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java index ad69d854e90..0fc52c01875 100644 --- a/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java +++ b/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java @@ -131,7 +131,7 @@ public class MapperQueryParser extends QueryParser { if (currentMapper != null) { Query query = null; if (currentMapper.useFieldQueryWithQueryString()) { - if (fieldMappers.hasDocMapper()) { + if (fieldMappers.explicitTypeInNameWithDocMapper()) { String[] previousTypes = QueryParseContext.setTypesWithPrevious(new String[]{fieldMappers.docMapper().type()}); try { query = currentMapper.fieldQuery(queryText, parseContext); @@ -202,7 +202,7 @@ public class MapperQueryParser extends QueryParser { if (currentMapper != null) { Query query = null; if (currentMapper.useFieldQueryWithQueryString()) { - if (fieldMappers.hasDocMapper()) { + if (fieldMappers.explicitTypeInNameWithDocMapper()) { String[] previousTypes = QueryParseContext.setTypesWithPrevious(new String[]{fieldMappers.docMapper().type()}); try { query = currentMapper.prefixQuery(termStr, multiTermRewriteMethod, parseContext); diff --git a/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/src/main/java/org/elasticsearch/index/mapper/MapperService.java index c65fe4ae801..241de4ad4f7 100644 --- a/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -742,6 +742,10 @@ public class MapperService extends AbstractIndexComponent implements Iterable Date: Mon, 30 Jan 2012 17:11:45 +0200 Subject: [PATCH 218/270] optimize bool filter for single must clause or single should clause --- .../common/lucene/search/XBooleanFilter.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/org/elasticsearch/common/lucene/search/XBooleanFilter.java b/src/main/java/org/elasticsearch/common/lucene/search/XBooleanFilter.java index 168d9b19afb..04f469cd860 100644 --- a/src/main/java/org/elasticsearch/common/lucene/search/XBooleanFilter.java +++ b/src/main/java/org/elasticsearch/common/lucene/search/XBooleanFilter.java @@ -70,6 +70,14 @@ public class XBooleanFilter extends Filter { public DocIdSet getDocIdSet(IndexReader reader) throws IOException { FixedBitSet res = null; + if (mustFilters == null && notFilters == null && shouldFilters != null && shouldFilters.size() == 1) { + return shouldFilters.get(0).getDocIdSet(reader); + } + + if (shouldFilters == null && notFilters == null && mustFilters != null && mustFilters.size() == 1) { + return mustFilters.get(0).getDocIdSet(reader); + } + if (shouldFilters != null) { for (int i = 0; i < shouldFilters.size(); i++) { final DocIdSet disi = getDISI(shouldFilters, i, reader); From 9194d36a64b3a00e1408c340c0305b201d05331c Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 30 Jan 2012 21:23:15 +0200 Subject: [PATCH 219/270] improve compressed string construction, allow to construct it from utf8 bytes and use it where applicable --- .../cluster/metadata/AliasMetaData.java | 3 ++- .../cluster/metadata/MappingMetaData.java | 4 +++- .../common/compress/CompressedString.java | 21 +++++++++++++++++++ .../index/mapper/DocumentMapper.java | 2 +- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java index f0f890b6813..1ac5f56e740 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java @@ -168,7 +168,8 @@ public class AliasMetaData { return this; } try { - this.filter = new CompressedString(XContentFactory.jsonBuilder().map(filter).string()); + XContentBuilder builder = XContentFactory.jsonBuilder().map(filter); + this.filter = new CompressedString(builder.underlyingBytes(), 0, builder.underlyingBytesLength()); return this; } catch (IOException e) { throw new ElasticSearchGenerationException("Failed to build json for alias request", e); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java index 15f9aeeeeb4..77ee0555581 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.joda.Joda; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; @@ -279,7 +280,8 @@ public class MappingMetaData { public MappingMetaData(String type, Map mapping) throws IOException { this.type = type; - this.source = new CompressedString(XContentFactory.jsonBuilder().map(mapping).string()); + XContentBuilder mappingBuilder = XContentFactory.jsonBuilder().map(mapping); + this.source = new CompressedString(mappingBuilder.underlyingBytes(), 0, mappingBuilder.underlyingBytesLength()); Map withoutType = mapping; if (mapping.size() == 1 && mapping.containsKey(type)) { withoutType = (Map) mapping.get(type); diff --git a/src/main/java/org/elasticsearch/common/compress/CompressedString.java b/src/main/java/org/elasticsearch/common/compress/CompressedString.java index b6230ade2ba..0fd564cd09b 100644 --- a/src/main/java/org/elasticsearch/common/compress/CompressedString.java +++ b/src/main/java/org/elasticsearch/common/compress/CompressedString.java @@ -21,6 +21,7 @@ package org.elasticsearch.common.compress; import org.apache.lucene.util.UnicodeUtil; import org.elasticsearch.common.Unicode; +import org.elasticsearch.common.compress.lzf.LZF; import org.elasticsearch.common.compress.lzf.LZFDecoder; import org.elasticsearch.common.compress.lzf.LZFEncoder; import org.elasticsearch.common.io.stream.StreamInput; @@ -40,10 +41,30 @@ public class CompressedString implements Streamable { CompressedString() { } + /** + * Constructor assuming the data provided is compressed (UTF8). It uses the provided + * array without copying it. + */ public CompressedString(byte[] compressed) { this.bytes = compressed; } + /** + * Constructs a new compressed string, assuming the bytes are UTF8, by copying it over. + * + * @param data The byte array + * @param offset Offset into the byte array + * @param length The length of the data + * @throws IOException + */ + public CompressedString(byte[] data, int offset, int length) throws IOException { + if (LZF.isCompressed(data, offset, length)) { + this.bytes = Arrays.copyOfRange(data, offset, offset + length); + } else { + this.bytes = LZFEncoder.encode(data, offset, length); + } + } + public CompressedString(String str) throws IOException { UnicodeUtil.UTF8Result result = Unicode.unsafeFromStringAsUtf8(str); this.bytes = LZFEncoder.encode(result.result, result.length); diff --git a/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index c06867044bc..41791d6dc78 100644 --- a/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -573,7 +573,7 @@ public class DocumentMapper implements ToXContent { builder.startObject(); toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); - this.mappingSource = new CompressedString(builder.string()); + this.mappingSource = new CompressedString(builder.underlyingBytes(), 0, builder.underlyingBytesLength()); } catch (Exception e) { throw new FailedToGenerateSourceMapperException(e.getMessage(), e); } From eb4f6709d97287b0c9de6af9bf5f4a42fcf98991 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 30 Jan 2012 22:39:54 +0200 Subject: [PATCH 220/270] Transport Client: Improve remote node freeze handling by adding another timeout layer, closes #1653. --- .../TransportClientNodesService.java | 95 ++++++++++++------- 1 file changed, 59 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java index c1ef7ddf6ad..ab4263082a7 100644 --- a/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java +++ b/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java @@ -55,6 +55,8 @@ public class TransportClientNodesService extends AbstractComponent { private final TimeValue nodesSamplerInterval; + private final long pingTimeout; + private final ClusterName clusterName; private final TransportService transportService; @@ -87,6 +89,7 @@ public class TransportClientNodesService extends AbstractComponent { this.threadPool = threadPool; this.nodesSamplerInterval = componentSettings.getAsTime("nodes_sampler_interval", timeValueSeconds(5)); + this.pingTimeout = componentSettings.getAsTime("ping_timeout", timeValueSeconds(5)).millis(); if (logger.isDebugEnabled()) { logger.debug("node_sampler_interval[" + nodesSamplerInterval + "]"); @@ -242,9 +245,13 @@ public class TransportClientNodesService extends AbstractComponent { class ScheduledNodeSampler implements Runnable { @Override public void run() { - nodesSampler.sample(); - if (!closed) { - nodesSamplerFuture = threadPool.schedule(nodesSamplerInterval, ThreadPool.Names.CACHED, this); + try { + nodesSampler.sample(); + if (!closed) { + nodesSamplerFuture = threadPool.schedule(nodesSamplerInterval, ThreadPool.Names.CACHED, this); + } + } catch (Exception e) { + logger.warn("failed to sample", e); } } } @@ -262,24 +269,28 @@ public class TransportClientNodesService extends AbstractComponent { try { transportService.connectToNode(node); } catch (Exception e) { - logger.debug("Failed to connect to node " + node + ", removed from nodes list", e); + logger.debug("failed to connect to node [{}], removed from nodes list", e, node); continue; } } try { - NodesInfoResponse nodeInfo = transportService.submitRequest(node, NodesInfoAction.NAME, Requests.nodesInfoRequest("_local"), new FutureTransportResponseHandler() { - @Override - public NodesInfoResponse newInstance() { - return new NodesInfoResponse(); - } - }).txGet(); + NodesInfoResponse nodeInfo = transportService.submitRequest(node, NodesInfoAction.NAME, + Requests.nodesInfoRequest("_local"), + TransportRequestOptions.options().withTimeout(pingTimeout), + new FutureTransportResponseHandler() { + @Override + public NodesInfoResponse newInstance() { + return new NodesInfoResponse(); + } + }).txGet(); if (!clusterName.equals(nodeInfo.clusterName())) { - logger.warn("Node {} not part of the cluster {}, ignoring...", node, clusterName); + logger.warn("node {} not part of the cluster {}, ignoring...", node, clusterName); } else { newNodes.add(node); } } catch (Exception e) { - logger.warn("failed to get node info for {}", e, node); + logger.info("failed to get node info for {}, disconnecting...", e, node); + transportService.disconnectFromNode(node); } } nodes = new ImmutableList.Builder().addAll(newNodes).build(); @@ -311,33 +322,45 @@ public class TransportClientNodesService extends AbstractComponent { @Override public void run() { try { - transportService.connectToNode(listedNode); // make sure we are connected to it - transportService.sendRequest(listedNode, NodesInfoAction.NAME, Requests.nodesInfoRequest("_all"), new BaseTransportResponseHandler() { - - @Override - public NodesInfoResponse newInstance() { - return new NodesInfoResponse(); + if (!transportService.nodeConnected(listedNode)) { + try { + transportService.connectToNode(listedNode); + } catch (Exception e) { + logger.debug("failed to connect to node [{}], removed from nodes list", e, listedNode); + return; } + } + transportService.sendRequest(listedNode, NodesInfoAction.NAME, + Requests.nodesInfoRequest("_all"), + TransportRequestOptions.options().withTimeout(pingTimeout), + new BaseTransportResponseHandler() { - @Override - public String executor() { - return ThreadPool.Names.SAME; - } + @Override + public NodesInfoResponse newInstance() { + return new NodesInfoResponse(); + } - @Override - public void handleResponse(NodesInfoResponse response) { - nodesInfoResponses.add(response); - latch.countDown(); - } + @Override + public String executor() { + return ThreadPool.Names.SAME; + } - @Override - public void handleException(TransportException exp) { - logger.debug("Failed to get node info from " + listedNode + ", removed from nodes list", exp); - latch.countDown(); - } - }); + @Override + public void handleResponse(NodesInfoResponse response) { + nodesInfoResponses.add(response); + latch.countDown(); + } + + @Override + public void handleException(TransportException e) { + logger.info("failed to get node info for {}, disconnecting...", e, listedNode); + transportService.disconnectFromNode(listedNode); + latch.countDown(); + } + }); } catch (Exception e) { - logger.debug("Failed to get node info from " + listedNode + ", removed from nodes list", e); + logger.info("failed to get node info for {}, disconnecting...", e, listedNode); + transportService.disconnectFromNode(listedNode); latch.countDown(); } } @@ -354,7 +377,7 @@ public class TransportClientNodesService extends AbstractComponent { for (NodesInfoResponse nodesInfoResponse : nodesInfoResponses) { for (NodeInfo nodeInfo : nodesInfoResponse) { if (!clusterName.equals(nodesInfoResponse.clusterName())) { - logger.warn("Node {} not part of the cluster {}, ignoring...", nodeInfo.node(), clusterName); + logger.warn("node {} not part of the cluster {}, ignoring...", nodeInfo.node(), clusterName); } else { if (nodeInfo.node().dataNode()) { // only add data nodes to connect to newNodes.add(nodeInfo.node()); @@ -369,7 +392,7 @@ public class TransportClientNodesService extends AbstractComponent { transportService.connectToNode(node); } catch (Exception e) { it.remove(); - logger.debug("Failed to connect to discovered node [" + node + "]", e); + logger.debug("failed to connect to discovered node [" + node + "]", e); } } nodes = new ImmutableList.Builder().addAll(newNodes).build(); From d4b6753d935d108ec0c5a60d41946755e6548242 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 31 Jan 2012 15:04:55 +0200 Subject: [PATCH 221/270] add a marker class to keep frame decoder around --- .../common/netty/KeepFrameDecoder.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/main/java/org/elasticsearch/common/netty/KeepFrameDecoder.java diff --git a/src/main/java/org/elasticsearch/common/netty/KeepFrameDecoder.java b/src/main/java/org/elasticsearch/common/netty/KeepFrameDecoder.java new file mode 100644 index 00000000000..f769f0fe0a4 --- /dev/null +++ b/src/main/java/org/elasticsearch/common/netty/KeepFrameDecoder.java @@ -0,0 +1,19 @@ +package org.elasticsearch.common.netty; + +import org.jboss.netty.buffer.ChannelBuffer; +import org.jboss.netty.channel.Channel; +import org.jboss.netty.channel.ChannelHandlerContext; +import org.jboss.netty.handler.codec.frame.FrameDecoder; + +/** + * A marker to not remove frame decoder from the resulting jar so plugins can use it. + */ +public class KeepFrameDecoder extends FrameDecoder { + + public static final KeepFrameDecoder decoder = new KeepFrameDecoder(); + + @Override + protected Object decode(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buffer) throws Exception { + return null; + } +} From 211176d10ee0e3992a63d73f71d1b863cf2e6568 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 31 Jan 2012 19:37:05 +0200 Subject: [PATCH 222/270] when no specific internal source builder is constructed, don't set it, so setting raw source will not get override --- .../org/elasticsearch/action/search/SearchRequestBuilder.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java index 7a69053243a..a289471d3ac 100644 --- a/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java @@ -716,7 +716,9 @@ public class SearchRequestBuilder extends BaseRequestBuilder listener) { - request.source(sourceBuilder()); + if (sourceBuilder != null) { + request.source(sourceBuilder()); + } client.search(request, listener); } From f282081361bd7b926805b4ac949ef9aca9affb38 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 31 Jan 2012 21:00:58 +0200 Subject: [PATCH 223/270] upgrade to latest compiled jsr166 libs --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 7ea73599120..3d18112dcfc 100644 --- a/pom.xml +++ b/pom.xml @@ -103,14 +103,14 @@ org.elasticsearch es-jsr166y - 20110918 + 20120131 compile org.elasticsearch es-jsr166e - 20110918 + 20120131 compile From e0508809cb08e7e03cc0a53bc7f77f9b08daeaee Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 1 Feb 2012 00:29:49 +0200 Subject: [PATCH 224/270] new local gateway should only load state if its data (for shards state) or master (for metadata) and not try and load if its not (so we don't load if its a client node) --- .../cluster/node/DiscoveryNode.java | 23 +++++++++++++++++-- .../state/meta/LocalGatewayMetaState.java | 19 ++++++++------- .../state/shards/LocalGatewayShardsState.java | 19 ++++++++------- 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java b/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java index 6338fc03a4f..df0a39391a5 100644 --- a/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java +++ b/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java @@ -37,8 +37,6 @@ import static org.elasticsearch.common.transport.TransportAddressSerializers.add /** * A discovery node represents a node that is part of the cluster. - * - * */ public class DiscoveryNode implements Streamable, Serializable { @@ -46,6 +44,27 @@ public class DiscoveryNode implements Streamable, Serializable { return !(settings.getAsBoolean("node.client", false) || (!settings.getAsBoolean("node.data", true) && !settings.getAsBoolean("node.master", true))); } + public static boolean clientNode(Settings settings) { + String client = settings.get("node.client"); + return client != null && client.equals("true"); + } + + public static boolean masterNode(Settings settings) { + String master = settings.get("node.master"); + if (master == null) { + return !clientNode(settings); + } + return master.equals("true"); + } + + public static boolean dataNode(Settings settings) { + String data = settings.get("data"); + if (data == null) { + return !clientNode(settings); + } + return data.equals("true"); + } + public static final ImmutableList EMPTY_LIST = ImmutableList.of(); private String nodeName = "".intern(); diff --git a/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java index daf32fdf1d5..2c58b8fc14d 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; @@ -73,14 +74,16 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS formatParams = ToXContent.EMPTY_PARAMS; } - try { - pre019Upgrade(); - long start = System.currentTimeMillis(); - loadState(); - logger.debug("took {} to load state", TimeValue.timeValueMillis(System.currentTimeMillis() - start)); - } catch (Exception e) { - logger.error("failed to read local state, exiting...", e); - throw e; + if (DiscoveryNode.masterNode(settings)) { + try { + pre019Upgrade(); + long start = System.currentTimeMillis(); + loadState(); + logger.debug("took {} to load state", TimeValue.timeValueMillis(System.currentTimeMillis() - start)); + } catch (Exception e) { + logger.error("failed to read local state, exiting...", e); + throw e; + } } } diff --git a/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java b/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java index 81ffef6a1f6..22c7c6fb700 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java @@ -23,6 +23,7 @@ import com.google.common.collect.Maps; import com.google.common.io.Closeables; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.*; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; @@ -62,14 +63,16 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste this.nodeEnv = nodeEnv; listGatewayStartedShards.initGateway(this); - try { - pre019Upgrade(); - long start = System.currentTimeMillis(); - loadStartedShards(); - logger.debug("took {} to load started shards state", TimeValue.timeValueMillis(System.currentTimeMillis() - start)); - } catch (Exception e) { - logger.error("failed to read local state (started shards), exiting...", e); - throw e; + if (DiscoveryNode.dataNode(settings)) { + try { + pre019Upgrade(); + long start = System.currentTimeMillis(); + loadStartedShards(); + logger.debug("took {} to load started shards state", TimeValue.timeValueMillis(System.currentTimeMillis() - start)); + } catch (Exception e) { + logger.error("failed to read local state (started shards), exiting...", e); + throw e; + } } } From e37b6ec2b7550a4437bf0b46f5f2f91643efdcc2 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 1 Feb 2012 11:49:47 +0200 Subject: [PATCH 225/270] improve text query to use term factory --- .../index/mapper/FieldMapper.java | 10 ++++++++ .../index/search/TextQueryParser.java | 23 ++++++++++--------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index ddec7dc6efe..854883c1c69 100644 --- a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -102,6 +102,16 @@ public interface FieldMapper { return sourcePath; } + /** + * The index name term that can be used as a factory. + */ + public Term indexNameTerm() { + return this.indexNameTermFactory; + } + + /** + * Creates a new index term based on the provided value. + */ public Term createIndexNameTerm(String value) { return indexNameTermFactory.createTerm(value); } diff --git a/src/main/java/org/elasticsearch/index/search/TextQueryParser.java b/src/main/java/org/elasticsearch/index/search/TextQueryParser.java index 48292ab1e20..227f9460d78 100644 --- a/src/main/java/org/elasticsearch/index/search/TextQueryParser.java +++ b/src/main/java/org/elasticsearch/index/search/TextQueryParser.java @@ -104,11 +104,13 @@ public class TextQueryParser { public Query parse(Type type) { FieldMapper mapper = null; - String field = fieldName; + Term fieldTerm; MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { mapper = smartNameFieldMappers.mapper(); - field = mapper.names().indexName(); + fieldTerm = mapper.names().indexNameTerm(); + } else { + fieldTerm = new Term(fieldName); } if (mapper != null && mapper.useFieldQueryWithQueryString()) { @@ -146,10 +148,10 @@ public class TextQueryParser { TokenStream source; try { - source = analyzer.reusableTokenStream(field, new FastStringReader(text)); + source = analyzer.reusableTokenStream(fieldTerm.field(), new FastStringReader(text)); source.reset(); } catch (IOException e) { - source = analyzer.tokenStream(field, new FastStringReader(text)); + source = analyzer.tokenStream(fieldTerm.field(), new FastStringReader(text)); } CachingTokenFilter buffer = new CachingTokenFilter(source); CharTermAttribute termAtt = null; @@ -203,7 +205,6 @@ public class TextQueryParser { // ignore } - Term termFactory = new Term(field); if (numTokens == 0) { return MatchNoDocsQuery.INSTANCE; } else if (type == Type.BOOLEAN) { @@ -216,7 +217,7 @@ public class TextQueryParser { } catch (IOException e) { // safe to ignore, because we know the number of tokens } - Query q = newTermQuery(mapper, termFactory.createTerm(term)); + Query q = newTermQuery(mapper, fieldTerm.createTerm(term)); return wrapSmartNameQuery(q, smartNameFieldMappers, parseContext); } BooleanQuery q = new BooleanQuery(positionCount == 1); @@ -230,7 +231,7 @@ public class TextQueryParser { // safe to ignore, because we know the number of tokens } - Query currentQuery = newTermQuery(mapper, termFactory.createTerm(term)); + Query currentQuery = newTermQuery(mapper, fieldTerm.createTerm(term)); q.add(currentQuery, occur); } return wrapSmartNameQuery(q, smartNameFieldMappers, parseContext); @@ -263,7 +264,7 @@ public class TextQueryParser { multiTerms.clear(); } position += positionIncrement; - multiTerms.add(termFactory.createTerm(term)); + multiTerms.add(fieldTerm.createTerm(term)); } if (enablePositionIncrements) { mpq.add(multiTerms.toArray(new Term[multiTerms.size()]), position); @@ -294,9 +295,9 @@ public class TextQueryParser { if (enablePositionIncrements) { position += positionIncrement; - pq.add(termFactory.createTerm(term), position); + pq.add(fieldTerm.createTerm(term), position); } else { - pq.add(termFactory.createTerm(term)); + pq.add(fieldTerm.createTerm(term)); } } return wrapSmartNameQuery(pq, smartNameFieldMappers, parseContext); @@ -330,7 +331,7 @@ public class TextQueryParser { multiTerms.clear(); } position += positionIncrement; - multiTerms.add(termFactory.createTerm(term)); + multiTerms.add(fieldTerm.createTerm(term)); } if (enablePositionIncrements) { mpq.add(multiTerms.toArray(new Term[multiTerms.size()]), position); From 70bf0a301b4131e63b698cccf44910cbf4570c1f Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 1 Feb 2012 11:52:57 +0200 Subject: [PATCH 226/270] have version implement serializable to DiscoveryNode can be serialized --- src/main/java/org/elasticsearch/Version.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/Version.java b/src/main/java/org/elasticsearch/Version.java index 1672a4b8ce4..2a8d7d8bdab 100644 --- a/src/main/java/org/elasticsearch/Version.java +++ b/src/main/java/org/elasticsearch/Version.java @@ -25,10 +25,11 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.monitor.jvm.JvmInfo; import java.io.IOException; +import java.io.Serializable; /** */ -public class Version { +public class Version implements Serializable { // The logic for ID is: XXYYZZAA, where XX is major version, YY is minor version, ZZ is revision, and AA is Beta/RC indicator // AA values below 50 are beta builds, and below 99 are RC builds, with 99 indicating a release From aa63811b14fed3b3ba1c7636f91c5ad6c7f37b62 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 1 Feb 2012 13:05:21 +0200 Subject: [PATCH 227/270] improve sycn on translog to happen on the snapshot thread instead of the schedule thread --- .../gateway/local/LocalIndexShardGateway.java | 41 +++++++++++++++---- .../index/translog/Translog.java | 2 + .../translog/fs/BufferingFsTranslogFile.java | 8 +++- .../index/translog/fs/FsTranslog.java | 6 +++ .../index/translog/fs/FsTranslogFile.java | 2 + .../translog/fs/SimpleFsTranslogFile.java | 6 +++ .../elasticsearch/threadpool/ThreadPool.java | 2 +- 7 files changed, 56 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java b/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java index 6955e0289a1..829bb4b3b8f 100644 --- a/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java +++ b/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java @@ -51,23 +51,26 @@ import java.util.concurrent.ScheduledFuture; */ public class LocalIndexShardGateway extends AbstractIndexShardComponent implements IndexShardGateway { + private final ThreadPool threadPool; + private final InternalIndexShard indexShard; private final RecoveryStatus recoveryStatus = new RecoveryStatus(); - private final ScheduledFuture flushScheduler; + private volatile ScheduledFuture flushScheduler; + private final TimeValue syncInterval; @Inject public LocalIndexShardGateway(ShardId shardId, @IndexSettings Settings indexSettings, ThreadPool threadPool, IndexShard indexShard) { super(shardId, indexSettings); + this.threadPool = threadPool; this.indexShard = (InternalIndexShard) indexShard; - TimeValue sync = componentSettings.getAsTime("sync", TimeValue.timeValueSeconds(10)); - if (sync.millis() > 0) { + syncInterval = componentSettings.getAsTime("sync", TimeValue.timeValueSeconds(5)); + if (syncInterval.millis() > 0) { this.indexShard.translog().syncOnEachOperation(false); - // we don't need to execute the sync on a different thread, just do it on the scheduler thread - flushScheduler = threadPool.scheduleWithFixedDelay(new Sync(), sync); - } else if (sync.millis() == 0) { + flushScheduler = threadPool.schedule(syncInterval, ThreadPool.Names.SAME, new Sync()); + } else if (syncInterval.millis() == 0) { flushScheduler = null; this.indexShard.translog().syncOnEachOperation(true); } else { @@ -237,11 +240,31 @@ public class LocalIndexShardGateway extends AbstractIndexShardComponent implemen return NO_SNAPSHOT_LOCK; } - private class Sync implements Runnable { + class Sync implements Runnable { @Override public void run() { - if (indexShard.state() == IndexShardState.STARTED) { - indexShard.translog().sync(); + // don't re-schedule if its closed..., we are done + if (indexShard.state() == IndexShardState.CLOSED) { + return; + } + if (indexShard.state() == IndexShardState.STARTED && indexShard.translog().syncNeeded()) { + threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() { + @Override + public void run() { + try { + indexShard.translog().sync(); + } catch (Exception e) { + if (indexShard.state() == IndexShardState.STARTED) { + logger.warn("failed to sync translog", e); + } + } + if (indexShard.state() != IndexShardState.CLOSED) { + flushScheduler = threadPool.schedule(syncInterval, ThreadPool.Names.SAME, Sync.this); + } + } + }); + } else { + flushScheduler = threadPool.schedule(syncInterval, ThreadPool.Names.SAME, Sync.this); } } } diff --git a/src/main/java/org/elasticsearch/index/translog/Translog.java b/src/main/java/org/elasticsearch/index/translog/Translog.java index 60b6850036e..0b8c06806b8 100644 --- a/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -117,6 +117,8 @@ public interface Translog extends IndexShardComponent { */ void sync(); + boolean syncNeeded(); + void syncOnEachOperation(boolean syncOnEachOperation); /** diff --git a/src/main/java/org/elasticsearch/index/translog/fs/BufferingFsTranslogFile.java b/src/main/java/org/elasticsearch/index/translog/fs/BufferingFsTranslogFile.java index 7809c78bf12..afa23dc5bdf 100644 --- a/src/main/java/org/elasticsearch/index/translog/fs/BufferingFsTranslogFile.java +++ b/src/main/java/org/elasticsearch/index/translog/fs/BufferingFsTranslogFile.java @@ -115,11 +115,16 @@ public class BufferingFsTranslogFile implements FsTranslogFile { } } + @Override + public boolean syncNeeded() { + return lastPosition != lastSyncPosition; + } + @Override public void sync() { try { // check if we really need to sync here... - long last = lastWrittenPosition; + long last = lastPosition; if (last == lastSyncPosition) { return; } @@ -142,6 +147,7 @@ public class BufferingFsTranslogFile implements FsTranslogFile { rwl.writeLock().lock(); try { flushBuffer(); + sync(); } catch (IOException e) { throw new TranslogException(shardId, "failed to close", e); } finally { diff --git a/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java b/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java index 23a4f6d6abd..dab85836481 100644 --- a/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java +++ b/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java @@ -382,6 +382,12 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog current1.sync(); } + @Override + public boolean syncNeeded() { + FsTranslogFile current1 = this.current; + return current1 != null && current1.syncNeeded(); + } + @Override public void syncOnEachOperation(boolean syncOnEachOperation) { this.syncOnEachOperation = syncOnEachOperation; diff --git a/src/main/java/org/elasticsearch/index/translog/fs/FsTranslogFile.java b/src/main/java/org/elasticsearch/index/translog/fs/FsTranslogFile.java index 68a70130310..eb7efd2e2a6 100644 --- a/src/main/java/org/elasticsearch/index/translog/fs/FsTranslogFile.java +++ b/src/main/java/org/elasticsearch/index/translog/fs/FsTranslogFile.java @@ -72,4 +72,6 @@ public interface FsTranslogFile { void reuse(FsTranslogFile other) throws TranslogException; void sync(); + + boolean syncNeeded(); } diff --git a/src/main/java/org/elasticsearch/index/translog/fs/SimpleFsTranslogFile.java b/src/main/java/org/elasticsearch/index/translog/fs/SimpleFsTranslogFile.java index 3a64f219d88..9b2ca7a7fc8 100644 --- a/src/main/java/org/elasticsearch/index/translog/fs/SimpleFsTranslogFile.java +++ b/src/main/java/org/elasticsearch/index/translog/fs/SimpleFsTranslogFile.java @@ -75,6 +75,7 @@ public class SimpleFsTranslogFile implements FsTranslogFile { } public void close(boolean delete) { + sync(); raf.decreaseRefCount(delete); } @@ -92,6 +93,11 @@ public class SimpleFsTranslogFile implements FsTranslogFile { } } + @Override + public boolean syncNeeded() { + return lastWrittenPosition.get() != lastSyncPosition; + } + public void sync() { try { // check if we really need to sync here... diff --git a/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index bbfac009112..bec358e73aa 100644 --- a/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -90,7 +90,7 @@ public class ThreadPool extends AbstractComponent { executors.put(Names.MANAGEMENT, build(Names.MANAGEMENT, "scaling", groupSettings.get(Names.MANAGEMENT), settingsBuilder().put("keep_alive", "5m").put("size", 20).build())); executors.put(Names.MERGE, build(Names.MERGE, "scaling", groupSettings.get(Names.MERGE), settingsBuilder().put("keep_alive", "5m").put("size", 20).build())); executors.put(Names.REFRESH, build(Names.REFRESH, "cached", groupSettings.get(Names.REFRESH), settingsBuilder().put("keep_alive", "1m").build())); - executors.put(Names.SNAPSHOT, build(Names.SNAPSHOT, "scaling", groupSettings.get(Names.SNAPSHOT), ImmutableSettings.Builder.EMPTY_SETTINGS)); + executors.put(Names.SNAPSHOT, build(Names.SNAPSHOT, "scaling", groupSettings.get(Names.SNAPSHOT), settingsBuilder().put("keep_alive", "5m").put("size", 5).build())); executors.put(Names.SAME, new ExecutorHolder(MoreExecutors.sameThreadExecutor(), new Info(Names.SAME, "same"))); this.executors = ImmutableMap.copyOf(executors); this.scheduler = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1, EsExecutors.daemonThreadFactory(settings, "[scheduler]")); From da062bb90596c925182bccd1ce1754f24a7e1314 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 1 Feb 2012 13:34:35 +0200 Subject: [PATCH 228/270] add node.max_local_storage_nodes to config template --- config/elasticsearch.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index 4789ccfd1e7..4746cb5c085 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -83,6 +83,10 @@ # node.rack: rack314 +# By default, multiple nodes are allowed to start from the same installation location +# to disable it, set the following: +# node.max_local_storage_nodes: 1 + #################################### Index #################################### # You can set a number of options (such as shard/replica options, mapping From 83e561d19f8aa2f3733db0ed159dc3f19f4b5272 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 1 Feb 2012 21:17:07 +0200 Subject: [PATCH 229/270] only update new versioned index or new indices when a new cluster state is received --- .../elasticsearch/discovery/zen/ZenDiscovery.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java index d1f1c72d47f..4faa8f6b004 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java @@ -24,6 +24,7 @@ import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ElasticSearchIllegalStateException; import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeService; @@ -492,6 +493,18 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen // same for metadata if (newState.metaData().version() == currentState.metaData().version()) { builder.metaData(currentState.metaData()); + } else { + // if its not the same version, only copy over new indices or ones that changed the version + MetaData.Builder metaDataBuilder = MetaData.builder().metaData(newState.metaData()).removeAllIndices(); + for (IndexMetaData indexMetaData : newState.metaData()) { + IndexMetaData currentIndexMetaData = currentState.metaData().index(indexMetaData.index()); + if (currentIndexMetaData == null || currentIndexMetaData.version() != indexMetaData.version()) { + metaDataBuilder.put(indexMetaData, false); + } else { + metaDataBuilder.put(currentIndexMetaData, false); + } + } + builder.metaData(metaDataBuilder); } return builder.build(); From a5838dc403fe48375790155bfdc4dbb7ce566437 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 1 Feb 2012 21:17:45 +0200 Subject: [PATCH 230/270] improve test, wait for green state post master node startup --- .../test/integration/cluster/MinimumMasterNodesTests.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/elasticsearch/test/integration/cluster/MinimumMasterNodesTests.java b/src/test/java/org/elasticsearch/test/integration/cluster/MinimumMasterNodesTests.java index b89d68fc9f8..0e8eb05892b 100644 --- a/src/test/java/org/elasticsearch/test/integration/cluster/MinimumMasterNodesTests.java +++ b/src/test/java/org/elasticsearch/test/integration/cluster/MinimumMasterNodesTests.java @@ -160,7 +160,9 @@ public class MinimumMasterNodesTests extends AbstractZenNodesTests { logger.info("--> starting the previous master node again..."); startNode(nonMasterNodeName, settings); - clusterHealthResponse = client("node1").admin().cluster().prepareHealth().setWaitForNodes("2").execute().actionGet(); + Thread.sleep(200); + + clusterHealthResponse = client("node1").admin().cluster().prepareHealth().setWaitForNodes("2").setWaitForGreenStatus().execute().actionGet(); assertThat(clusterHealthResponse.timedOut(), equalTo(false)); state = client("node1").admin().cluster().prepareState().setLocal(true).execute().actionGet().state(); From 3698bcc2b921cf2cda5e8009f01f22b969399a94 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 2 Feb 2012 00:18:17 +0200 Subject: [PATCH 231/270] better logging of recovery --- .../gateway/IndexShardGatewayService.java | 1 + .../index/gateway/RecoveryStatus.java | 37 +++++++++++++++++++ .../blobstore/BlobStoreIndexShardGateway.java | 19 +++++++--- .../gateway/local/LocalIndexShardGateway.java | 17 +++++++-- .../shard/service/InternalIndexShard.java | 9 +++++ .../indices/recovery/RecoveryResponse.java | 4 ++ .../indices/recovery/RecoverySource.java | 9 +++-- .../indices/recovery/RecoveryTarget.java | 3 +- 8 files changed, 85 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/gateway/IndexShardGatewayService.java b/src/main/java/org/elasticsearch/index/gateway/IndexShardGatewayService.java index e4173afe0ea..179e7ba1ae6 100644 --- a/src/main/java/org/elasticsearch/index/gateway/IndexShardGatewayService.java +++ b/src/main/java/org/elasticsearch/index/gateway/IndexShardGatewayService.java @@ -197,6 +197,7 @@ public class IndexShardGatewayService extends AbstractIndexShardComponent implem sb.append(" index : files [").append(recoveryStatus.index().numberOfFiles()).append("] with total_size [").append(new ByteSizeValue(recoveryStatus.index().totalSize())).append("], took[").append(TimeValue.timeValueMillis(recoveryStatus.index().time())).append("]\n"); sb.append(" : recovered_files [").append(recoveryStatus.index().numberOfRecoveredFiles()).append("] with total_size [").append(new ByteSizeValue(recoveryStatus.index().recoveredTotalSize())).append("]\n"); sb.append(" : reusing_files [").append(recoveryStatus.index().numberOfReusedFiles()).append("] with total_size [").append(new ByteSizeValue(recoveryStatus.index().reusedTotalSize())).append("]\n"); + sb.append(" start : took [").append(TimeValue.timeValueMillis(recoveryStatus.start().time())).append("], check_index [").append(timeValueMillis(recoveryStatus.start().checkIndexTime())).append("]\n"); sb.append(" translog : number_of_operations [").append(recoveryStatus.translog().currentTranslogOperations()).append("], took [").append(TimeValue.timeValueMillis(recoveryStatus.translog().time())).append("]"); logger.debug(sb.toString()); } diff --git a/src/main/java/org/elasticsearch/index/gateway/RecoveryStatus.java b/src/main/java/org/elasticsearch/index/gateway/RecoveryStatus.java index 0b761af9f65..a8fe3d1af23 100644 --- a/src/main/java/org/elasticsearch/index/gateway/RecoveryStatus.java +++ b/src/main/java/org/elasticsearch/index/gateway/RecoveryStatus.java @@ -29,6 +29,7 @@ public class RecoveryStatus { public static enum Stage { INIT, INDEX, + START, TRANSLOG, DONE } @@ -43,6 +44,8 @@ public class RecoveryStatus { private Translog translog = new Translog(); + private Start start = new Start(); + public Stage stage() { return this.stage; } @@ -72,10 +75,44 @@ public class RecoveryStatus { return index; } + public Start start() { + return this.start; + } + public Translog translog() { return translog; } + public static class Start { + private long startTime; + private long time; + private long checkIndexTime; + + public long startTime() { + return this.startTime; + } + + public void startTime(long startTime) { + this.startTime = startTime; + } + + public long time() { + return this.time; + } + + public void time(long time) { + this.time = time; + } + + public long checkIndexTime() { + return checkIndexTime; + } + + public void checkIndexTime(long checkIndexTime) { + this.checkIndexTime = checkIndexTime; + } + } + public static class Translog { private long startTime = 0; private long time; diff --git a/src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java b/src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java index 45fcb2af431..13235e592e8 100644 --- a/src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java +++ b/src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java @@ -405,8 +405,6 @@ public abstract class BlobStoreIndexShardGateway extends AbstractIndexShardCompo } recoveryStatus.index().startTime(System.currentTimeMillis()); recoveryStatus.index().time(System.currentTimeMillis() - recoveryStatus.index().startTime()); - recoveryStatus.translog().startTime(System.currentTimeMillis()); - recoveryStatus.translog().time(System.currentTimeMillis() - recoveryStatus.index().startTime()); return; } @@ -417,14 +415,10 @@ public abstract class BlobStoreIndexShardGateway extends AbstractIndexShardCompo } try { recoveryStatus.index().startTime(System.currentTimeMillis()); - recoveryStatus.updateStage(RecoveryStatus.Stage.INDEX); recoverIndex(commitPoint, blobs); recoveryStatus.index().time(System.currentTimeMillis() - recoveryStatus.index().startTime()); - recoveryStatus.translog().startTime(System.currentTimeMillis()); - recoveryStatus.updateStage(RecoveryStatus.Stage.TRANSLOG); recoverTranslog(commitPoint, blobs); - recoveryStatus.translog().time(System.currentTimeMillis() - recoveryStatus.index().startTime()); return; } catch (Exception e) { throw new IndexShardGatewayRecoveryException(shardId, "failed to recover commit_point [" + commitPoint.name() + "]/[" + commitPoint.version() + "]", e); @@ -436,12 +430,23 @@ public abstract class BlobStoreIndexShardGateway extends AbstractIndexShardCompo private void recoverTranslog(CommitPoint commitPoint, ImmutableMap blobs) throws IndexShardGatewayRecoveryException { if (commitPoint.translogFiles().isEmpty()) { // no translog files, bail + recoveryStatus.start().startTime(System.currentTimeMillis()); + recoveryStatus.updateStage(RecoveryStatus.Stage.START); indexShard.start("post recovery from gateway, no translog"); + recoveryStatus.start().time(System.currentTimeMillis() - recoveryStatus.start().startTime()); + recoveryStatus.start().checkIndexTime(indexShard.checkIndexTook()); return; } try { + recoveryStatus.start().startTime(System.currentTimeMillis()); + recoveryStatus.updateStage(RecoveryStatus.Stage.START); indexShard.performRecoveryPrepareForTranslog(); + recoveryStatus.start().time(System.currentTimeMillis() - recoveryStatus.start().startTime()); + recoveryStatus.start().checkIndexTime(indexShard.checkIndexTook()); + + recoveryStatus.updateStage(RecoveryStatus.Stage.TRANSLOG); + recoveryStatus.translog().startTime(System.currentTimeMillis()); final AtomicReference failure = new AtomicReference(); final CountDownLatch latch = new CountDownLatch(1); @@ -529,12 +534,14 @@ public abstract class BlobStoreIndexShardGateway extends AbstractIndexShardCompo } indexShard.performRecoveryFinalization(true); + recoveryStatus.translog().time(System.currentTimeMillis() - recoveryStatus.translog().startTime()); } catch (Throwable e) { throw new IndexShardGatewayRecoveryException(shardId, "Failed to recover translog", e); } } private void recoverIndex(CommitPoint commitPoint, ImmutableMap blobs) throws Exception { + recoveryStatus.updateStage(RecoveryStatus.Stage.INDEX); int numberOfFiles = 0; long totalSize = 0; int numberOfReusedFiles = 0; diff --git a/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java b/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java index 829bb4b3b8f..2839a0b7d2e 100644 --- a/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java +++ b/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java @@ -91,6 +91,7 @@ public class LocalIndexShardGateway extends AbstractIndexShardComponent implemen @Override public void recover(boolean indexShouldExists, RecoveryStatus recoveryStatus) throws IndexShardGatewayRecoveryException { recoveryStatus.index().startTime(System.currentTimeMillis()); + recoveryStatus.updateStage(RecoveryStatus.Stage.INDEX); long version = -1; long translogId = -1; try { @@ -124,12 +125,14 @@ public class LocalIndexShardGateway extends AbstractIndexShardComponent implemen // ignore } - recoveryStatus.translog().startTime(System.currentTimeMillis()); + recoveryStatus.start().startTime(System.currentTimeMillis()); + recoveryStatus.updateStage(RecoveryStatus.Stage.START); if (translogId == -1) { // no translog files, bail indexShard.start("post recovery from gateway, no translog"); // no index, just start the shard and bail - recoveryStatus.translog().time(System.currentTimeMillis() - recoveryStatus.index().startTime()); + recoveryStatus.start().time(System.currentTimeMillis() - recoveryStatus.start().startTime()); + recoveryStatus.start().checkIndexTime(indexShard.checkIndexTook()); return; } @@ -163,12 +166,18 @@ public class LocalIndexShardGateway extends AbstractIndexShardComponent implemen // no translog files, bail indexShard.start("post recovery from gateway, no translog"); // no index, just start the shard and bail - recoveryStatus.translog().time(System.currentTimeMillis() - recoveryStatus.index().startTime()); + recoveryStatus.start().time(System.currentTimeMillis() - recoveryStatus.start().startTime()); + recoveryStatus.start().checkIndexTime(indexShard.checkIndexTook()); return; } // recover from the translog file indexShard.performRecoveryPrepareForTranslog(); + recoveryStatus.start().time(System.currentTimeMillis() - recoveryStatus.start().startTime()); + recoveryStatus.start().checkIndexTime(indexShard.checkIndexTook()); + + recoveryStatus.translog().startTime(System.currentTimeMillis()); + recoveryStatus.updateStage(RecoveryStatus.Stage.TRANSLOG); try { InputStreamStreamInput si = new InputStreamStreamInput(new FileInputStream(recoveringTranslogFile)); while (true) { @@ -195,7 +204,7 @@ public class LocalIndexShardGateway extends AbstractIndexShardComponent implemen recoveringTranslogFile.delete(); - recoveryStatus.translog().time(System.currentTimeMillis() - recoveryStatus.index().startTime()); + recoveryStatus.translog().time(System.currentTimeMillis() - recoveryStatus.translog().startTime()); } @Override diff --git a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java index 768281d77e5..c439a912d09 100644 --- a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java +++ b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java @@ -115,6 +115,8 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I private final boolean checkIndexOnStartup; + private long checkIndexTook = 0; + private volatile IndexShardState state; private TimeValue refreshInterval; @@ -550,6 +552,10 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I } } + public long checkIndexTook() { + return this.checkIndexTook; + } + /** * After the store has been recovered, we need to start the engine in order to apply operations */ @@ -830,6 +836,8 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I private void checkIndex(boolean throwException) throws IndexShardException { try { + checkIndexTook = 0; + long time = System.currentTimeMillis(); if (!IndexReader.indexExists(store.directory())) { return; } @@ -853,6 +861,7 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I logger.debug("check index [success]\n{}", new String(os.underlyingBytes(), 0, os.size())); } } + checkIndexTook = System.currentTimeMillis() - time; } catch (Exception e) { logger.warn("failed to check index", e); } diff --git a/src/main/java/org/elasticsearch/indices/recovery/RecoveryResponse.java b/src/main/java/org/elasticsearch/indices/recovery/RecoveryResponse.java index 82c317d46e4..51cd49cb299 100644 --- a/src/main/java/org/elasticsearch/indices/recovery/RecoveryResponse.java +++ b/src/main/java/org/elasticsearch/indices/recovery/RecoveryResponse.java @@ -41,6 +41,8 @@ class RecoveryResponse implements Streamable { long phase1Time; long phase1ThrottlingWaitTime; + long startTime; + int phase2Operations; long phase2Time; @@ -78,6 +80,7 @@ class RecoveryResponse implements Streamable { phase1ExistingTotalSize = in.readVLong(); phase1Time = in.readVLong(); phase1ThrottlingWaitTime = in.readVLong(); + startTime = in.readVLong(); phase2Operations = in.readVInt(); phase2Time = in.readVLong(); phase3Operations = in.readVInt(); @@ -108,6 +111,7 @@ class RecoveryResponse implements Streamable { out.writeVLong(phase1ExistingTotalSize); out.writeVLong(phase1Time); out.writeVLong(phase1ThrottlingWaitTime); + out.writeVLong(startTime); out.writeVInt(phase2Operations); out.writeVLong(phase2Time); out.writeVInt(phase3Operations); diff --git a/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java b/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java index b7028026695..fec99a6c009 100644 --- a/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java +++ b/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java @@ -194,13 +194,16 @@ public class RecoverySource extends AbstractComponent { if (shard.state() == IndexShardState.CLOSED) { throw new IndexShardClosedException(request.shardId()); } - logger.trace("[{}][{}] recovery [phase2] to {}: sending transaction log operations", request.shardId().index().name(), request.shardId().id(), request.targetNode()); + logger.trace("[{}][{}] recovery [phase2] to {}: start", request.shardId().index().name(), request.shardId().id(), request.targetNode()); StopWatch stopWatch = new StopWatch().start(); - transportService.submitRequest(request.targetNode(), RecoveryTarget.Actions.PREPARE_TRANSLOG, new RecoveryPrepareForTranslogOperationsRequest(request.shardId()), VoidTransportResponseHandler.INSTANCE_SAME).txGet(); + stopWatch.stop(); + response.startTime = stopWatch.totalTime().millis(); + logger.trace("[{}][{}] recovery [phase2] to {}: start took [{}]", request.shardId().index().name(), request.shardId().id(), request.targetNode(), stopWatch.totalTime()); + logger.trace("[{}][{}] recovery [phase2] to {}: sending transaction log operations", request.shardId().index().name(), request.shardId().id(), request.targetNode()); + stopWatch = new StopWatch().start(); int totalOperations = sendSnapshot(snapshot); - stopWatch.stop(); logger.trace("[{}][{}] recovery [phase2] to {}: took [{}]", request.shardId().index().name(), request.shardId().id(), request.targetNode(), stopWatch.totalTime()); response.phase2Time = stopWatch.totalTime().millis(); diff --git a/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java index 1ce4036377d..dd300cf6492 100644 --- a/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java +++ b/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java @@ -195,7 +195,8 @@ public class RecoveryTarget extends AbstractComponent { .append(", took [").append(timeValueMillis(recoveryStatus.phase1Time)).append("], throttling_wait [").append(timeValueMillis(recoveryStatus.phase1ThrottlingWaitTime)).append(']') .append("\n"); sb.append(" : reusing_files [").append(recoveryStatus.phase1ExistingFileNames.size()).append("] with total_size of [").append(new ByteSizeValue(recoveryStatus.phase1ExistingTotalSize)).append("]\n"); - sb.append(" phase2: recovered [").append(recoveryStatus.phase2Operations).append("]").append(" transaction log operations") + sb.append(" phase2: start took [").append(timeValueMillis(recoveryStatus.startTime)).append("]\n"); + sb.append(" : recovered [").append(recoveryStatus.phase2Operations).append("]").append(" transaction log operations") .append(", took [").append(timeValueMillis(recoveryStatus.phase2Time)).append("]") .append("\n"); sb.append(" phase3: recovered [").append(recoveryStatus.phase3Operations).append("]").append(" transaction log operations") From 8b9c75cd7eb510bb9c3153e7bdf6abc166958073 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 2 Feb 2012 00:35:51 +0200 Subject: [PATCH 232/270] when executing on local node, don't reset the shard iterator, its null... --- .../single/custom/TransportSingleCustomOperationAction.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/action/support/single/custom/TransportSingleCustomOperationAction.java b/src/main/java/org/elasticsearch/action/support/single/custom/TransportSingleCustomOperationAction.java index edb47af79b8..61c22bc1f41 100644 --- a/src/main/java/org/elasticsearch/action/support/single/custom/TransportSingleCustomOperationAction.java +++ b/src/main/java/org/elasticsearch/action/support/single/custom/TransportSingleCustomOperationAction.java @@ -143,7 +143,6 @@ public abstract class TransportSingleCustomOperationAction Date: Thu, 2 Feb 2012 00:49:34 +0200 Subject: [PATCH 233/270] Query: Add `index.query.default_field` allowing to control the default field used to search on, closes #1657. --- .../admin/indices/analyze/TransportAnalyzeAction.java | 7 ++++++- .../index/query/FuzzyLikeThisQueryParser.java | 3 +-- .../index/query/IndexQueryParserService.java | 9 +++++++++ .../index/query/MoreLikeThisQueryParser.java | 3 +-- .../org/elasticsearch/index/query/QueryParseContext.java | 4 ++++ .../index/query/QueryStringQueryParser.java | 3 +-- 6 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index 24b696795d5..0bf769cc8e6 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -41,6 +41,7 @@ import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.analysis.*; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.internal.AllFieldMapper; import org.elasticsearch.index.service.IndexService; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.analysis.IndicesAnalysisService; @@ -130,7 +131,11 @@ public class TransportAnalyzeAction extends TransportSingleCustomOperationAction } } if (field == null) { - field = "_all"; + if (indexService != null) { + field = indexService.queryParserService().defaultField(); + } else { + field = AllFieldMapper.NAME; + } } if (analyzer == null && request.analyzer() != null) { if (indexService == null) { diff --git a/src/main/java/org/elasticsearch/index/query/FuzzyLikeThisQueryParser.java b/src/main/java/org/elasticsearch/index/query/FuzzyLikeThisQueryParser.java index 60c2c8898c3..cdfc49a50f9 100644 --- a/src/main/java/org/elasticsearch/index/query/FuzzyLikeThisQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/FuzzyLikeThisQueryParser.java @@ -25,7 +25,6 @@ import org.apache.lucene.search.FuzzyLikeThisQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.internal.AllFieldMapper; import java.io.IOException; import java.util.List; @@ -114,7 +113,7 @@ public class FuzzyLikeThisQueryParser implements QueryParser { FuzzyLikeThisQuery query = new FuzzyLikeThisQuery(maxNumTerms, analyzer); if (fields == null) { // add the default _all field - query.addTerms(likeText, AllFieldMapper.NAME, minSimilarity, prefixLength); + query.addTerms(likeText, parseContext.defaultField(), minSimilarity, prefixLength); } else { for (String field : fields) { query.addTerms(likeText, field, minSimilarity, prefixLength); diff --git a/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java b/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java index 142a2d805fd..d4fbd991317 100644 --- a/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java +++ b/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.cache.IndexCache; import org.elasticsearch.index.engine.IndexEngine; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.internal.AllFieldMapper; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.query.IndicesQueriesRegistry; @@ -81,6 +82,8 @@ public class IndexQueryParserService extends AbstractIndexComponent { private final Map filterParsers; + private String defaultField; + @Inject public IndexQueryParserService(Index index, @IndexSettings Settings indexSettings, IndicesQueriesRegistry indicesQueriesRegistry, @@ -97,6 +100,8 @@ public class IndexQueryParserService extends AbstractIndexComponent { this.indexCache = indexCache; this.indexEngine = indexEngine; + this.defaultField = indexSettings.get("index.query.default_field", AllFieldMapper.NAME); + List queryParsers = newArrayList(); if (namedQueryParsers != null) { Map queryParserGroups = indexSettings.getGroups(IndexQueryParserService.Defaults.QUERY_PREFIX); @@ -148,6 +153,10 @@ public class IndexQueryParserService extends AbstractIndexComponent { cache.remove(); } + public String defaultField() { + return this.defaultField; + } + public QueryParser queryParser(String name) { return queryParsers.get(name); } diff --git a/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryParser.java b/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryParser.java index 63e2269da0f..3a5afcdca50 100644 --- a/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryParser.java @@ -26,7 +26,6 @@ import org.apache.lucene.search.Query; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.MoreLikeThisQuery; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.internal.AllFieldMapper; import java.io.IOException; import java.util.List; @@ -53,7 +52,7 @@ public class MoreLikeThisQueryParser implements QueryParser { XContentParser parser = parseContext.parser(); MoreLikeThisQuery mltQuery = new MoreLikeThisQuery(); - mltQuery.setMoreLikeFields(new String[]{AllFieldMapper.NAME}); + mltQuery.setMoreLikeFields(new String[]{parseContext.defaultField()}); mltQuery.setSimilarity(parseContext.searchSimilarity()); Analyzer analyzer = null; diff --git a/src/main/java/org/elasticsearch/index/query/QueryParseContext.java b/src/main/java/org/elasticsearch/index/query/QueryParseContext.java index f2902a844bd..dcbc286625b 100644 --- a/src/main/java/org/elasticsearch/index/query/QueryParseContext.java +++ b/src/main/java/org/elasticsearch/index/query/QueryParseContext.java @@ -130,6 +130,10 @@ public class QueryParseContext { return indexQueryParser.indexCache; } + public String defaultField() { + return indexQueryParser.defaultField(); + } + public MapperQueryParser singleQueryParser(QueryParserSettings settings) { queryParser.reset(settings); return queryParser; diff --git a/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java b/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java index 04b55bfea74..9187f3c44d2 100644 --- a/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.internal.AllFieldMapper; import org.elasticsearch.index.query.support.QueryParsers; import java.io.IOException; @@ -68,7 +67,7 @@ public class QueryStringQueryParser implements QueryParser { XContentParser parser = parseContext.parser(); MultiFieldQueryParserSettings qpSettings = new MultiFieldQueryParserSettings(); - qpSettings.defaultField(AllFieldMapper.NAME); + qpSettings.defaultField(parseContext.defaultField()); qpSettings.analyzeWildcard(defaultAnalyzeWildcard); qpSettings.allowLeadingWildcard(defaultAllowLeadingWildcard); From 07ba39740bb0e7037301f53935ad0425a1365e18 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 2 Feb 2012 12:45:20 +0200 Subject: [PATCH 234/270] use non deprecated methods --- src/main/java/org/elasticsearch/http/HttpServer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/http/HttpServer.java b/src/main/java/org/elasticsearch/http/HttpServer.java index efc87166cf6..2b66c33cbc5 100644 --- a/src/main/java/org/elasticsearch/http/HttpServer.java +++ b/src/main/java/org/elasticsearch/http/HttpServer.java @@ -90,12 +90,12 @@ public class HttpServer extends AbstractLifecycleComponent { if (logger.isInfoEnabled()) { logger.info("{}", transport.boundAddress()); } - nodeService.putNodeAttribute("http_address", transport.boundAddress().publishAddress().toString()); + nodeService.putAttribute("http_address", transport.boundAddress().publishAddress().toString()); } @Override protected void doStop() throws ElasticSearchException { - nodeService.removeNodeAttribute("http_address"); + nodeService.removeAttribute("http_address"); transport.stop(); } From 43af5041c38adc5f07300509459c8fa79450a268 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 2 Feb 2012 13:26:12 +0200 Subject: [PATCH 235/270] Allow for plugins to register REST pre processor, closes #1658. --- .../org/elasticsearch/http/HttpServer.java | 13 ++++-- .../elasticsearch/rest/RestController.java | 41 +++++++++++++++-- .../elasticsearch/rest/RestPreProcessor.java | 45 +++++++++++++++++++ 3 files changed, 91 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/elasticsearch/rest/RestPreProcessor.java diff --git a/src/main/java/org/elasticsearch/http/HttpServer.java b/src/main/java/org/elasticsearch/http/HttpServer.java index 2b66c33cbc5..6ad5b876404 100644 --- a/src/main/java/org/elasticsearch/http/HttpServer.java +++ b/src/main/java/org/elasticsearch/http/HttpServer.java @@ -27,10 +27,7 @@ import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.node.service.NodeService; -import org.elasticsearch.rest.BytesRestResponse; -import org.elasticsearch.rest.RestController; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.StringRestResponse; +import org.elasticsearch.rest.*; import java.io.File; import java.io.IOException; @@ -114,6 +111,14 @@ public class HttpServer extends AbstractLifecycleComponent { public void internalDispatchRequest(final HttpRequest request, final HttpChannel channel) { if (request.rawPath().startsWith("/_plugin/")) { + for (RestPreProcessor preProcessor : restController.preProcessors()) { + if (!preProcessor.handleExternal()) { + continue; + } + if (!preProcessor.process(request, channel)) { + return; + } + } handlePluginSite(request, channel); return; } diff --git a/src/main/java/org/elasticsearch/rest/RestController.java b/src/main/java/org/elasticsearch/rest/RestController.java index 6906afe8638..30926d0fdba 100644 --- a/src/main/java/org/elasticsearch/rest/RestController.java +++ b/src/main/java/org/elasticsearch/rest/RestController.java @@ -28,6 +28,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.support.RestUtils; import java.io.IOException; +import java.util.Arrays; +import java.util.Comparator; /** * @@ -41,6 +43,9 @@ public class RestController extends AbstractLifecycleComponent { private final PathTrie headHandlers = new PathTrie(RestUtils.REST_DECODER); private final PathTrie optionsHandlers = new PathTrie(RestUtils.REST_DECODER); + // non volatile since the assumption is that pre processors are registered on startup + private RestPreProcessor[] preProcessors = new RestPreProcessor[0]; + @Inject public RestController(Settings settings) { super(settings); @@ -58,6 +63,25 @@ public class RestController extends AbstractLifecycleComponent { protected void doClose() throws ElasticSearchException { } + /** + * Registers a pre processor to be executed before the rest request is actually handled. + */ + public synchronized void registerPreProcessor(RestPreProcessor preProcessor) { + RestPreProcessor[] copy = new RestPreProcessor[preProcessors.length + 1]; + System.arraycopy(preProcessors, 0, copy, 0, preProcessors.length); + copy[preProcessors.length] = preProcessor; + Arrays.sort(copy, new Comparator() { + @Override + public int compare(RestPreProcessor o1, RestPreProcessor o2) { + return o2.order() - o1.order(); + } + }); + preProcessors = copy; + } + + /** + * Registers a rest handler to be execute when the provided method and path match the request. + */ public void registerHandler(RestRequest.Method method, String path, RestHandler handler) { switch (method) { case GET: @@ -83,12 +107,21 @@ public class RestController extends AbstractLifecycleComponent { } } + public RestPreProcessor[] preProcessors() { + return preProcessors; + } + public boolean dispatchRequest(final RestRequest request, final RestChannel channel) { - final RestHandler handler = getHandler(request); - if (handler == null) { - return false; - } try { + for (RestPreProcessor preProcessor : preProcessors) { + if (!preProcessor.process(request, channel)) { + return true; + } + } + final RestHandler handler = getHandler(request); + if (handler == null) { + return false; + } handler.handleRequest(request, channel); } catch (Exception e) { try { diff --git a/src/main/java/org/elasticsearch/rest/RestPreProcessor.java b/src/main/java/org/elasticsearch/rest/RestPreProcessor.java new file mode 100644 index 00000000000..1494454eebf --- /dev/null +++ b/src/main/java/org/elasticsearch/rest/RestPreProcessor.java @@ -0,0 +1,45 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest; + +/** + * Rest pre processor allowing to pre process REST requests. + *

    + * Experimental interface. + */ +public interface RestPreProcessor { + + /** + * Optionally, the order the processor will work on. Execution is done from lowest value to highest. + * It is a good practice to allow to configure this for the relevant processor. + */ + int order(); + + /** + * Should this processor also process external (non REST) requests, like plugin site requests. + */ + boolean handleExternal(); + + /** + * Process the request, returning false if no further processing should be done. Note, + * make sure to send a response if returning false, otherwise, no response will be sent. + */ + boolean process(RestRequest request, RestChannel channel); +} From a784f9fcdf33f1bb63ffbe699365efc9cfc21ed4 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 2 Feb 2012 13:47:59 +0200 Subject: [PATCH 236/270] add a note on the rest pre processor --- src/main/java/org/elasticsearch/rest/RestPreProcessor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/elasticsearch/rest/RestPreProcessor.java b/src/main/java/org/elasticsearch/rest/RestPreProcessor.java index 1494454eebf..273fbd5ca19 100644 --- a/src/main/java/org/elasticsearch/rest/RestPreProcessor.java +++ b/src/main/java/org/elasticsearch/rest/RestPreProcessor.java @@ -40,6 +40,9 @@ public interface RestPreProcessor { /** * Process the request, returning false if no further processing should be done. Note, * make sure to send a response if returning false, otherwise, no response will be sent. + *

    + * It is recommended that the process method will not do blocking calls, or heavily cache data + * if a blocking call is done. */ boolean process(RestRequest request, RestChannel channel); } From 81e6ff51623792ebe9cb7c277fccd044bdd432b9 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 2 Feb 2012 20:19:15 +0200 Subject: [PATCH 237/270] Allow for plugins to register REST filter (better support with async execution and some renaming), closes #1658. --- .../org/elasticsearch/http/HttpServer.java | 38 +++--- .../elasticsearch/rest/RestController.java | 126 ++++++++++++++---- ...{RestPreProcessor.java => RestFilter.java} | 36 ++--- .../elasticsearch/rest/RestFilterChain.java | 32 +++++ 4 files changed, 171 insertions(+), 61 deletions(-) rename src/main/java/org/elasticsearch/rest/{RestPreProcessor.java => RestFilter.java} (53%) create mode 100644 src/main/java/org/elasticsearch/rest/RestFilterChain.java diff --git a/src/main/java/org/elasticsearch/http/HttpServer.java b/src/main/java/org/elasticsearch/http/HttpServer.java index 6ad5b876404..0e5fb51f0ff 100644 --- a/src/main/java/org/elasticsearch/http/HttpServer.java +++ b/src/main/java/org/elasticsearch/http/HttpServer.java @@ -51,6 +51,8 @@ public class HttpServer extends AbstractLifecycleComponent { private final boolean disableSites; + private final PluginSiteFilter pluginSiteFilter = new PluginSiteFilter(); + @Inject public HttpServer(Settings settings, Environment environment, HttpServerTransport transport, RestController restController, @@ -111,33 +113,33 @@ public class HttpServer extends AbstractLifecycleComponent { public void internalDispatchRequest(final HttpRequest request, final HttpChannel channel) { if (request.rawPath().startsWith("/_plugin/")) { - for (RestPreProcessor preProcessor : restController.preProcessors()) { - if (!preProcessor.handleExternal()) { - continue; - } - if (!preProcessor.process(request, channel)) { - return; - } - } - handlePluginSite(request, channel); + RestFilterChain filterChain = restController.filterChain(pluginSiteFilter); + filterChain.continueProcessing(request, channel); return; } - if (!restController.dispatchRequest(request, channel)) { - if (request.method() == RestRequest.Method.OPTIONS) { - // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added) - StringRestResponse response = new StringRestResponse(OK); - channel.sendResponse(response); - } else { - channel.sendResponse(new StringRestResponse(BAD_REQUEST, "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]")); - } + restController.dispatchRequest(request, channel); + } + + + class PluginSiteFilter extends RestFilter { + + @Override + public void process(RestRequest request, RestChannel channel, RestFilterChain filterChain) { + handlePluginSite((HttpRequest) request, (HttpChannel) channel); } } - private void handlePluginSite(HttpRequest request, HttpChannel channel) { + void handlePluginSite(HttpRequest request, HttpChannel channel) { if (disableSites) { channel.sendResponse(new StringRestResponse(FORBIDDEN)); return; } + if (request.method() == RestRequest.Method.OPTIONS) { + // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added) + StringRestResponse response = new StringRestResponse(OK); + channel.sendResponse(response); + return; + } if (request.method() != RestRequest.Method.GET) { channel.sendResponse(new StringRestResponse(FORBIDDEN)); return; diff --git a/src/main/java/org/elasticsearch/rest/RestController.java b/src/main/java/org/elasticsearch/rest/RestController.java index 30926d0fdba..e852e76c2f5 100644 --- a/src/main/java/org/elasticsearch/rest/RestController.java +++ b/src/main/java/org/elasticsearch/rest/RestController.java @@ -21,6 +21,8 @@ package org.elasticsearch.rest; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.ElasticSearchIllegalStateException; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.path.PathTrie; @@ -31,6 +33,9 @@ import java.io.IOException; import java.util.Arrays; import java.util.Comparator; +import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; +import static org.elasticsearch.rest.RestStatus.OK; + /** * */ @@ -43,8 +48,10 @@ public class RestController extends AbstractLifecycleComponent { private final PathTrie headHandlers = new PathTrie(RestUtils.REST_DECODER); private final PathTrie optionsHandlers = new PathTrie(RestUtils.REST_DECODER); + private final RestHandlerFilter handlerFilter = new RestHandlerFilter(); + // non volatile since the assumption is that pre processors are registered on startup - private RestPreProcessor[] preProcessors = new RestPreProcessor[0]; + private RestFilter[] filters = new RestFilter[0]; @Inject public RestController(Settings settings) { @@ -61,22 +68,25 @@ public class RestController extends AbstractLifecycleComponent { @Override protected void doClose() throws ElasticSearchException { + for (RestFilter filter : filters) { + filter.close(); + } } /** * Registers a pre processor to be executed before the rest request is actually handled. */ - public synchronized void registerPreProcessor(RestPreProcessor preProcessor) { - RestPreProcessor[] copy = new RestPreProcessor[preProcessors.length + 1]; - System.arraycopy(preProcessors, 0, copy, 0, preProcessors.length); - copy[preProcessors.length] = preProcessor; - Arrays.sort(copy, new Comparator() { + public synchronized void registerFilter(RestFilter preProcessor) { + RestFilter[] copy = new RestFilter[filters.length + 1]; + System.arraycopy(filters, 0, copy, 0, filters.length); + copy[filters.length] = preProcessor; + Arrays.sort(copy, new Comparator() { @Override - public int compare(RestPreProcessor o1, RestPreProcessor o2) { + public int compare(RestFilter o1, RestFilter o2) { return o2.order() - o1.order(); } }); - preProcessors = copy; + filters = copy; } /** @@ -107,30 +117,55 @@ public class RestController extends AbstractLifecycleComponent { } } - public RestPreProcessor[] preProcessors() { - return preProcessors; + /** + * Returns a filter chain (if needed) to execute. If this method returns null, simply execute + * as usual. + */ + @Nullable + public RestFilterChain filterChainOrNull(RestFilter executionFilter) { + if (filters.length == 0) { + return null; + } + return new ControllerFilterChain(executionFilter); } - public boolean dispatchRequest(final RestRequest request, final RestChannel channel) { - try { - for (RestPreProcessor preProcessor : preProcessors) { - if (!preProcessor.process(request, channel)) { - return true; + /** + * Returns a filter chain with the final filter being the provided filter. + */ + public RestFilterChain filterChain(RestFilter executionFilter) { + return new ControllerFilterChain(executionFilter); + } + + public void dispatchRequest(final RestRequest request, final RestChannel channel) { + if (filters.length == 0) { + try { + executeHandler(request, channel); + } catch (Exception e) { + try { + channel.sendResponse(new XContentThrowableRestResponse(request, e)); + } catch (IOException e1) { + logger.error("Failed to send failure response for uri [" + request.uri() + "]", e1); } } - final RestHandler handler = getHandler(request); - if (handler == null) { - return false; - } + } else { + ControllerFilterChain filterChain = new ControllerFilterChain(handlerFilter); + filterChain.continueProcessing(request, channel); + } + } + + void executeHandler(RestRequest request, RestChannel channel) { + final RestHandler handler = getHandler(request); + if (handler != null) { handler.handleRequest(request, channel); - } catch (Exception e) { - try { - channel.sendResponse(new XContentThrowableRestResponse(request, e)); - } catch (IOException e1) { - logger.error("Failed to send failure response for uri [" + request.uri() + "]", e1); + } else { + if (request.method() == RestRequest.Method.OPTIONS) { + // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added) + StringRestResponse response = new StringRestResponse(OK); + channel.sendResponse(response); + } else { + channel.sendResponse(new StringRestResponse(BAD_REQUEST, "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]")); } } - return true; } private RestHandler getHandler(RestRequest request) { @@ -159,4 +194,45 @@ public class RestController extends AbstractLifecycleComponent { // my_index/my_type/http%3A%2F%2Fwww.google.com return request.rawPath(); } + + class ControllerFilterChain implements RestFilterChain { + + private final RestFilter executionFilter; + + private volatile int index; + + ControllerFilterChain(RestFilter executionFilter) { + this.executionFilter = executionFilter; + } + + @Override + public void continueProcessing(RestRequest request, RestChannel channel) { + try { + int loc = index; + if (loc > filters.length) { + throw new ElasticSearchIllegalStateException("filter continueProcessing was called more than expected"); + } else if (loc == filters.length) { + executionFilter.process(request, channel, this); + } else { + RestFilter preProcessor = filters[loc]; + preProcessor.process(request, channel, this); + } + index++; + } catch (Exception e) { + try { + channel.sendResponse(new XContentThrowableRestResponse(request, e)); + } catch (IOException e1) { + logger.error("Failed to send failure response for uri [" + request.uri() + "]", e1); + } + } + } + } + + class RestHandlerFilter extends RestFilter { + + @Override + public void process(RestRequest request, RestChannel channel, RestFilterChain filterChain) { + executeHandler(request, channel); + } + } } diff --git a/src/main/java/org/elasticsearch/rest/RestPreProcessor.java b/src/main/java/org/elasticsearch/rest/RestFilter.java similarity index 53% rename from src/main/java/org/elasticsearch/rest/RestPreProcessor.java rename to src/main/java/org/elasticsearch/rest/RestFilter.java index 273fbd5ca19..cf71e9f9734 100644 --- a/src/main/java/org/elasticsearch/rest/RestPreProcessor.java +++ b/src/main/java/org/elasticsearch/rest/RestFilter.java @@ -19,30 +19,30 @@ package org.elasticsearch.rest; +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.component.CloseableComponent; + /** - * Rest pre processor allowing to pre process REST requests. - *

    - * Experimental interface. + * A filter allowing to filter rest operations. */ -public interface RestPreProcessor { +public abstract class RestFilter implements CloseableComponent { /** - * Optionally, the order the processor will work on. Execution is done from lowest value to highest. - * It is a good practice to allow to configure this for the relevant processor. + * Optionally, the order of the filter. Execution is done from lowest value to highest. + * It is a good practice to allow to configure this for the relevant filter. */ - int order(); + public int order() { + return 0; + } + + @Override + public void close() throws ElasticSearchException { + // a no op + } /** - * Should this processor also process external (non REST) requests, like plugin site requests. + * Process the rest request. Using the channel to send a response, or the filter chain to continue + * processing the request. */ - boolean handleExternal(); - - /** - * Process the request, returning false if no further processing should be done. Note, - * make sure to send a response if returning false, otherwise, no response will be sent. - *

    - * It is recommended that the process method will not do blocking calls, or heavily cache data - * if a blocking call is done. - */ - boolean process(RestRequest request, RestChannel channel); + public abstract void process(RestRequest request, RestChannel channel, RestFilterChain filterChain); } diff --git a/src/main/java/org/elasticsearch/rest/RestFilterChain.java b/src/main/java/org/elasticsearch/rest/RestFilterChain.java new file mode 100644 index 00000000000..564e43af6f4 --- /dev/null +++ b/src/main/java/org/elasticsearch/rest/RestFilterChain.java @@ -0,0 +1,32 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest; + +/** + * A filter chain allowing to continue and process the rest request. + */ +public interface RestFilterChain { + + /** + * Continue processing the request. Should only be called if a response has not been sent + * through the channel. + */ + void continueProcessing(RestRequest request, RestChannel channel); +} From a6a3e0039d2789850085376b913726b3e809f0f8 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 2 Feb 2012 20:47:06 +0200 Subject: [PATCH 238/270] return 403 when thread pool is overloaded and can't add a request to the queue --- .../elasticsearch/threadpool/ThreadPool.java | 25 ++++++++++++- .../ThreadPoolRejectedException.java | 37 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/elasticsearch/threadpool/ThreadPoolRejectedException.java diff --git a/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index bec358e73aa..7f7c4737a6f 100644 --- a/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -216,7 +216,7 @@ public class ThreadPool extends AbstractComponent { RejectedExecutionHandler rejectedExecutionHandler; String rejectSetting = settings.get("reject_policy", defaultSettings.get("reject_policy", "abort")); if ("abort".equals(rejectSetting)) { - rejectedExecutionHandler = new ThreadPoolExecutor.AbortPolicy(); + rejectedExecutionHandler = new AbortPolicy(); } else if ("caller".equals(rejectSetting)) { rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy(); } else { @@ -350,6 +350,29 @@ public class ThreadPool extends AbstractComponent { } } + /** + * A handler for rejected tasks that throws a + * RejectedExecutionException. + */ + public static class AbortPolicy implements RejectedExecutionHandler { + /** + * Creates an AbortPolicy. + */ + public AbortPolicy() { + } + + /** + * Always throws RejectedExecutionException. + * + * @param r the runnable task requested to be executed + * @param e the executor attempting to execute this task + * @throws RejectedExecutionException always. + */ + public void rejectedExecution(Runnable r, ThreadPoolExecutor e) { + throw new ThreadPoolRejectedException(); + } + } + static class ExecutorHolder { public final Executor executor; public final Info info; diff --git a/src/main/java/org/elasticsearch/threadpool/ThreadPoolRejectedException.java b/src/main/java/org/elasticsearch/threadpool/ThreadPoolRejectedException.java new file mode 100644 index 00000000000..914bcf3c227 --- /dev/null +++ b/src/main/java/org/elasticsearch/threadpool/ThreadPoolRejectedException.java @@ -0,0 +1,37 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.threadpool; + +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.rest.RestStatus; + +/** + */ +public class ThreadPoolRejectedException extends ElasticSearchException { + + public ThreadPoolRejectedException() { + super("rejected"); + } + + @Override + public RestStatus status() { + return RestStatus.FORBIDDEN; + } +} From 445bed43584383eb3a413939b3b08a272bad9cc9 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 2 Feb 2012 21:02:42 +0200 Subject: [PATCH 239/270] fix wrong name --- src/main/resources/config/names.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/resources/config/names.txt b/src/main/resources/config/names.txt index 0bfc04495b9..2b18b99c623 100644 --- a/src/main/resources/config/names.txt +++ b/src/main/resources/config/names.txt @@ -861,7 +861,8 @@ Eros Eshu Eson the Searcher Miguel Espinosa -[[Mister Sinister|Essex, Nathaniel +Mister Sinister +Essex, Nathaniel Eternal Brain Eternity Ev Teel Urizen From 68f838a4880246dcb9c472cc26e0621d6092184e Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 3 Feb 2012 10:01:07 +0200 Subject: [PATCH 240/270] fs flag should be properly serialized in node stats --- .../action/admin/cluster/node/stats/NodesStatsRequest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java index 0e9f0648e93..df3443133c3 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -211,6 +211,7 @@ public class NodesStatsRequest extends NodesOperationRequest { jvm = in.readBoolean(); threadPool = in.readBoolean(); network = in.readBoolean(); + fs = in.readBoolean(); transport = in.readBoolean(); http = in.readBoolean(); } @@ -224,6 +225,7 @@ public class NodesStatsRequest extends NodesOperationRequest { out.writeBoolean(jvm); out.writeBoolean(threadPool); out.writeBoolean(network); + out.writeBoolean(fs); out.writeBoolean(transport); out.writeBoolean(http); } From e06de6b24c6f8531bd85de2707cd05dd4c7d8051 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 3 Feb 2012 14:20:42 +0200 Subject: [PATCH 241/270] only check if settings need to change or mappings need to be checked if the index metadata has actually changed --- .../cluster/ClusterChangedEvent.java | 16 ++++++++++++++++ .../cluster/IndicesClusterStateService.java | 11 +++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java b/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java index 6fb0c84d6f6..b4b8ff85408 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java @@ -21,6 +21,8 @@ package org.elasticsearch.cluster; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; import java.util.List; @@ -122,6 +124,20 @@ public class ClusterChangedEvent { return state.metaData() != previousState.metaData(); } + public boolean indexMetaDataChanged(IndexMetaData current) { + MetaData previousMetaData = previousState.metaData(); + if (previousMetaData == null) { + return true; + } + IndexMetaData previousIndexMetaData = previousMetaData.index(current.index()); + // no need to check on version, since disco modules will make sure to use the + // same instance if its a version match + if (previousIndexMetaData == current) { + return false; + } + return false; + } + public boolean blocksChanged() { return state.blocks() != previousState.blocks(); } diff --git a/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index 1f2d5271798..2829ecd2ef1 100644 --- a/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -304,11 +304,18 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent typesToRefresh = null; String index = indexMetaData.index(); IndexService indexService = indicesService.indexServiceSafe(index); From 3ba2dfe9cad35da239693c87dd51d7a4349683ff Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 3 Feb 2012 16:11:58 +0200 Subject: [PATCH 242/270] revert back applying mapping changes on index changes --- .../indices/cluster/IndicesClusterStateService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index 2829ecd2ef1..9bb2b6908df 100644 --- a/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -332,9 +332,9 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent typesToRefresh = null; String index = indexMetaData.index(); IndexService indexService = indicesService.indexServiceSafe(index); From 1aca5861b9b43ec7ff561e4debe1ad5e52750b18 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 3 Feb 2012 16:19:04 +0200 Subject: [PATCH 243/270] checking on index metadata does not make sense for mappings... --- .../indices/cluster/IndicesClusterStateService.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index 9bb2b6908df..6e2493ec8d0 100644 --- a/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -331,10 +331,6 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent typesToRefresh = null; String index = indexMetaData.index(); IndexService indexService = indicesService.indexServiceSafe(index); From 6c406d0e0ed532d5b7cfeaeb01b78dcb675cbf0d Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 5 Feb 2012 12:04:13 +0200 Subject: [PATCH 244/270] Indices / Nodes stats: All `all` flag to easily return all stats, closes #1662. --- .../cluster/node/stats/NodesStatsRequest.java | 16 ++++++++++++++++ .../node/stats/NodesStatsRequestBuilder.java | 8 ++++++++ .../indices/stats/IndicesStatsRequest.java | 17 +++++++++++++++++ .../stats/IndicesStatsRequestBuilder.java | 11 +++++++++++ .../node/stats/RestNodesStatsAction.java | 4 ++++ .../indices/stats/RestIndicesStatsAction.java | 4 ++++ 6 files changed, 60 insertions(+) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java index df3443133c3..36018f18e0e 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -51,6 +51,22 @@ public class NodesStatsRequest extends NodesOperationRequest { super(nodesIds); } + /** + * Sets all the request flags. + */ + public NodesStatsRequest all() { + this.indices = true; + this.os = true; + this.process = true; + this.jvm = true; + this.threadPool = true; + this.network = true; + this.fs = true; + this.transport = true; + this.http = true; + return this; + } + /** * Clears all the request flags. */ diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java index dbca5dfa3a0..cf126290c7c 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java @@ -37,6 +37,14 @@ public class NodesStatsRequestBuilder extends BaseClusterRequestBuilder Date: Sun, 5 Feb 2012 13:06:58 +0200 Subject: [PATCH 245/270] Query DSL: query_string/field/text do not fail when an invalid analyzer is provided, closes #1663. --- .../org/elasticsearch/index/query/FieldQueryParser.java | 7 ++++++- .../elasticsearch/index/query/QueryStringQueryParser.java | 7 ++++++- .../org/elasticsearch/index/query/TextQueryParser.java | 5 ++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java b/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java index c331e1e6386..be37ad04595 100644 --- a/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/FieldQueryParser.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.support.QueryParsers; import java.io.IOException; @@ -93,7 +94,11 @@ public class FieldQueryParser implements QueryParser { } else if ("phrase_slop".equals(currentFieldName) || "phraseSlop".equals(currentFieldName)) { qpSettings.phraseSlop(parser.intValue()); } else if ("analyzer".equals(currentFieldName)) { - qpSettings.forcedAnalyzer(parseContext.analysisService().analyzer(parser.text())); + NamedAnalyzer analyzer = parseContext.analysisService().analyzer(parser.text()); + if (analyzer == null) { + throw new QueryParsingException(parseContext.index(), "[query_string] analyzer [" + parser.text() + "] not found"); + } + qpSettings.forcedAnalyzer(analyzer); } else if ("default_operator".equals(currentFieldName) || "defaultOperator".equals(currentFieldName)) { String op = parser.text(); if ("or".equalsIgnoreCase(op)) { diff --git a/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java b/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java index 9187f3c44d2..9e5ef29f808 100644 --- a/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.support.QueryParsers; import java.io.IOException; @@ -136,7 +137,11 @@ public class QueryStringQueryParser implements QueryParser { throw new QueryParsingException(parseContext.index(), "Query default operator [" + op + "] is not allowed"); } } else if ("analyzer".equals(currentFieldName)) { - qpSettings.forcedAnalyzer(parseContext.analysisService().analyzer(parser.text())); + NamedAnalyzer analyzer = parseContext.analysisService().analyzer(parser.text()); + if (analyzer == null) { + throw new QueryParsingException(parseContext.index(), "[query_string] analyzer [" + parser.text() + "] not found"); + } + qpSettings.forcedAnalyzer(analyzer); } else if ("allow_leading_wildcard".equals(currentFieldName) || "allowLeadingWildcard".equals(currentFieldName)) { qpSettings.allowLeadingWildcard(parser.booleanValue()); } else if ("auto_generate_phrase_queries".equals(currentFieldName) || "autoGeneratePhraseQueries".equals(currentFieldName)) { diff --git a/src/main/java/org/elasticsearch/index/query/TextQueryParser.java b/src/main/java/org/elasticsearch/index/query/TextQueryParser.java index b8042b49029..fa636db93e0 100644 --- a/src/main/java/org/elasticsearch/index/query/TextQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/TextQueryParser.java @@ -88,7 +88,10 @@ public class TextQueryParser implements QueryParser { type = org.elasticsearch.index.search.TextQueryParser.Type.PHRASE_PREFIX; } } else if ("analyzer".equals(currentFieldName)) { - analyzer = parser.textOrNull(); + analyzer = parser.text(); + if (parseContext.analysisService().analyzer(analyzer) == null) { + throw new QueryParsingException(parseContext.index(), "[text] analyzer [" + parser.text() + "] not found"); + } } else if ("boost".equals(currentFieldName)) { boost = parser.floatValue(); } else if ("slop".equals(currentFieldName) || "phrase_slop".equals(currentFieldName) || "phraseSlop".equals(currentFieldName)) { From 3cee291bc2d9af7177f51804e82fca0663c8341a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 5 Feb 2012 13:45:32 +0200 Subject: [PATCH 246/270] Index / Delete API: timeout should apply automatically to the auto create index API master timeout, closes #1664. --- .../action/admin/indices/create/CreateIndexRequest.java | 9 +++++++++ .../action/delete/TransportDeleteAction.java | 2 +- .../elasticsearch/action/index/TransportIndexAction.java | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index eb88f013532..f1f2aec4f74 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -251,6 +251,15 @@ public class CreateIndexRequest extends MasterNodeOperationRequest { return timeout(TimeValue.parseTimeValue(timeout, null)); } + /** + * A timeout value in case the master has not been discovered yet or disconnected. + */ + @Override + public CreateIndexRequest masterNodeTimeout(TimeValue timeout) { + this.masterNodeTimeout = timeout; + return this; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); diff --git a/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java b/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java index 22854eae3f8..3692a947933 100644 --- a/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java +++ b/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java @@ -75,7 +75,7 @@ public class TransportDeleteAction extends TransportShardReplicationOperationAct protected void doExecute(final DeleteRequest request, final ActionListener listener) { if (autoCreateIndex && !clusterService.state().metaData().hasConcreteIndex(request.index())) { request.beforeLocalFork(); - createIndexAction.execute(new CreateIndexRequest(request.index()), new ActionListener() { + createIndexAction.execute(new CreateIndexRequest(request.index()).cause("auto(delete api)").masterNodeTimeout(request.timeout()), new ActionListener() { @Override public void onResponse(CreateIndexResponse result) { innerExecute(request, listener); diff --git a/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java b/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java index 416b569dc0b..c7d92a75b3e 100644 --- a/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java +++ b/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java @@ -93,7 +93,7 @@ public class TransportIndexAction extends TransportShardReplicationOperationActi // if we don't have a master, we don't have metadata, that's fine, let it find a master using create index API if (autoCreateIndex && !clusterService.state().metaData().hasConcreteIndex(request.index())) { request.beforeLocalFork(); // we fork on another thread... - createIndexAction.execute(new CreateIndexRequest(request.index()).cause("auto(index api)"), new ActionListener() { + createIndexAction.execute(new CreateIndexRequest(request.index()).cause("auto(index api)").masterNodeTimeout(request.timeout()), new ActionListener() { @Override public void onResponse(CreateIndexResponse result) { innerExecute(request, listener); From 1c2fa366c914b16b23f8681598578ff7fefda448 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 5 Feb 2012 17:46:58 +0200 Subject: [PATCH 247/270] Node Stats: JVM stats to provide memory pools allocations, closes #1665. --- .../elasticsearch/monitor/jvm/JvmStats.java | 145 +++++++++++++++++- 1 file changed, 140 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java b/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java index a466a9c8625..c7b348081f2 100644 --- a/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java +++ b/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java @@ -103,6 +103,20 @@ public class JvmStats implements Streamable, Serializable, ToXContent { stats.mem.nonHeapUsed = memUsage.getUsed() < 0 ? 0 : memUsage.getUsed(); stats.mem.nonHeapCommitted = memUsage.getCommitted() < 0 ? 0 : memUsage.getCommitted(); + List memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans(); + stats.mem.pools = new MemoryPool[memoryPoolMXBeans.size()]; + for (int i = 0; i < memoryPoolMXBeans.size(); i++) { + MemoryPoolMXBean memoryPoolMXBean = memoryPoolMXBeans.get(i); + MemoryUsage usage = memoryPoolMXBean.getUsage(); + MemoryUsage peakUsage = memoryPoolMXBean.getPeakUsage(); + stats.mem.pools[i] = new MemoryPool(memoryPoolMXBean.getName(), + usage.getUsed() < 0 ? 0 : usage.getUsed(), + usage.getMax() < 0 ? 0 : usage.getMax(), + peakUsage.getUsed() < 0 ? 0 : peakUsage.getUsed(), + peakUsage.getMax() < 0 ? 0 : peakUsage.getMax() + ); + } + stats.threads = new Threads(); stats.threads.count = threadMXBean.getThreadCount(); stats.threads.peakCount = threadMXBean.getPeakThreadCount(); @@ -221,9 +235,27 @@ public class JvmStats implements Streamable, Serializable, ToXContent { builder.field("heap_committed_in_bytes", mem.heapCommitted().bytes()); builder.field("non_heap_used", mem.nonHeapUsed().toString()); - builder.field("non_heap_used_in_bytes", mem.nonHeapUsed().bytes()); + builder.field("non_heap_used_in_bytes", mem.nonHeapUsed); builder.field("non_heap_committed", mem.nonHeapCommitted().toString()); - builder.field("non_heap_committed_in_bytes", mem.nonHeapCommitted().bytes()); + builder.field("non_heap_committed_in_bytes", mem.nonHeapCommitted); + + builder.startObject("pools"); + for (MemoryPool pool : mem) { + builder.startObject(pool.name()); + builder.field("used", pool.used().toString()); + builder.field("used_in_bytes", pool.used); + builder.field("max", pool.max().toString()); + builder.field("max_in_bytes", pool.max); + + builder.field("peak_used", pool.peakUsed().toString()); + builder.field("peak_used_in_bytes", pool.peakUsed); + builder.field("peak_max", pool.peakMax().toString()); + builder.field("peak_max_in_bytes", pool.peakMax); + + builder.endObject(); + } + builder.endObject(); + builder.endObject(); } if (threads != null) { @@ -282,7 +314,7 @@ public class JvmStats implements Streamable, Serializable, ToXContent { public static class GarbageCollectors implements Streamable, Serializable, Iterable { - private GarbageCollector[] collectors; + GarbageCollector[] collectors; GarbageCollectors() { } @@ -314,7 +346,7 @@ public class JvmStats implements Streamable, Serializable, ToXContent { } @Override - public Iterator iterator() { + public Iterator iterator() { return Iterators.forArray(collectors); } @@ -554,13 +586,101 @@ public class JvmStats implements Streamable, Serializable, ToXContent { } } - public static class Mem implements Streamable, Serializable { + public static class MemoryPool implements Streamable, Serializable { + + String name; + long used; + long max; + + long peakUsed; + long peakMax; + + MemoryPool() { + + } + + public MemoryPool(String name, long used, long max, long peakUsed, long peakMax) { + this.name = name; + this.used = used; + this.max = max; + this.peakUsed = peakUsed; + this.peakMax = peakMax; + } + + public static MemoryPool readMemoryPool(StreamInput in) throws IOException { + MemoryPool pool = new MemoryPool(); + pool.readFrom(in); + return pool; + } + + public String name() { + return this.name; + } + + public String getName() { + return this.name; + } + + public ByteSizeValue used() { + return new ByteSizeValue(used); + } + + public ByteSizeValue getUsed() { + return used(); + } + + public ByteSizeValue max() { + return new ByteSizeValue(max); + } + + public ByteSizeValue getMax() { + return max(); + } + + public ByteSizeValue peakUsed() { + return new ByteSizeValue(peakUsed); + } + + public ByteSizeValue getPeakUsed() { + return peakUsed(); + } + + public ByteSizeValue peakMax() { + return new ByteSizeValue(peakMax); + } + + public ByteSizeValue getPeakMax() { + return peakMax(); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + name = in.readUTF(); + used = in.readVLong(); + max = in.readVLong(); + peakUsed = in.readVLong(); + peakMax = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeUTF(name); + out.writeVLong(used); + out.writeVLong(max); + out.writeVLong(peakUsed); + out.writeVLong(peakMax); + } + } + + public static class Mem implements Streamable, Serializable, Iterable { long heapCommitted; long heapUsed; long nonHeapCommitted; long nonHeapUsed; + MemoryPool[] pools = new MemoryPool[0]; + Mem() { } @@ -570,12 +690,22 @@ public class JvmStats implements Streamable, Serializable, ToXContent { return mem; } + @Override + public Iterator iterator() { + return Iterators.forArray(pools); + } + @Override public void readFrom(StreamInput in) throws IOException { heapCommitted = in.readVLong(); heapUsed = in.readVLong(); nonHeapCommitted = in.readVLong(); nonHeapUsed = in.readVLong(); + + pools = new MemoryPool[in.readVInt()]; + for (int i = 0; i < pools.length; i++) { + pools[i] = MemoryPool.readMemoryPool(in); + } } @Override @@ -584,6 +714,11 @@ public class JvmStats implements Streamable, Serializable, ToXContent { out.writeVLong(heapUsed); out.writeVLong(nonHeapCommitted); out.writeVLong(nonHeapUsed); + + out.writeVInt(pools.length); + for (MemoryPool pool : pools) { + pool.writeTo(out); + } } public ByteSizeValue heapCommitted() { From 80607dd7c3cc8dfc1d393c38d9ab204d5ec683c2 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 5 Feb 2012 18:06:01 +0200 Subject: [PATCH 248/270] remove unused variable --- .../cluster/routing/allocation/AllocationModule.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationModule.java b/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationModule.java index 2d81c915c77..3e338c65718 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationModule.java +++ b/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationModule.java @@ -20,17 +20,13 @@ package org.elasticsearch.cluster.routing.allocation; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocatorModule; -import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecidersModule; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.inject.SpawnModules; import org.elasticsearch.common.settings.Settings; -import java.util.List; - /** * */ @@ -38,8 +34,6 @@ public class AllocationModule extends AbstractModule implements SpawnModules { private final Settings settings; - private List> allocations = Lists.newArrayList(); - public AllocationModule(Settings settings) { this.settings = settings; } From 44a604029366f5983de181eeb2ac54bd1ad3b219 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 5 Feb 2012 18:52:56 +0200 Subject: [PATCH 249/270] Jmx: Only register JMX beans when jmx.create_connector is set to `true`, or explicitly set by setting `jmx.export` to true, closes #1666. --- .../org/elasticsearch/index/IndexModule.java | 12 ++++- .../index/service/InternalIndexService.java | 2 +- .../index/shard/IndexShardModule.java | 11 ++++- .../index/store/StoreModule.java | 5 ++- .../indices/InternalIndicesService.java | 2 +- .../elasticsearch/jmx/AbstractJmxModule.java | 45 +++++++++++++++++++ .../java/org/elasticsearch/jmx/JmxModule.java | 4 +- .../org/elasticsearch/jmx/JmxService.java | 5 +++ .../transport/TransportModule.java | 5 ++- .../transport/local/LocalTransportModule.java | 12 ++++- .../transport/netty/NettyTransportModule.java | 12 ++++- 11 files changed, 105 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/elasticsearch/jmx/AbstractJmxModule.java diff --git a/src/main/java/org/elasticsearch/index/IndexModule.java b/src/main/java/org/elasticsearch/index/IndexModule.java index ea22d837b46..0052628c035 100644 --- a/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/src/main/java/org/elasticsearch/index/IndexModule.java @@ -20,17 +20,27 @@ package org.elasticsearch.index; import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.service.IndexService; import org.elasticsearch.index.service.InternalIndexService; +import org.elasticsearch.jmx.JmxService; /** * */ public class IndexModule extends AbstractModule { + private final Settings settings; + + public IndexModule(Settings settings) { + this.settings = settings; + } + @Override protected void configure() { bind(IndexService.class).to(InternalIndexService.class).asEagerSingleton(); - bind(IndexServiceManagement.class).asEagerSingleton(); + if (JmxService.shouldExport(settings)) { + bind(IndexServiceManagement.class).asEagerSingleton(); + } } } diff --git a/src/main/java/org/elasticsearch/index/service/InternalIndexService.java b/src/main/java/org/elasticsearch/index/service/InternalIndexService.java index 0b11b974350..7fcac7d983b 100644 --- a/src/main/java/org/elasticsearch/index/service/InternalIndexService.java +++ b/src/main/java/org/elasticsearch/index/service/InternalIndexService.java @@ -295,7 +295,7 @@ public class InternalIndexService extends AbstractIndexComponent implements Inde ModulesBuilder modules = new ModulesBuilder(); modules.add(new ShardsPluginsModule(indexSettings, pluginsService)); - modules.add(new IndexShardModule(shardId)); + modules.add(new IndexShardModule(indexSettings, shardId)); modules.add(new ShardIndexingModule()); modules.add(new ShardSearchModule()); modules.add(new ShardGetModule()); diff --git a/src/main/java/org/elasticsearch/index/shard/IndexShardModule.java b/src/main/java/org/elasticsearch/index/shard/IndexShardModule.java index b39a7760d99..e4a055d6fd3 100644 --- a/src/main/java/org/elasticsearch/index/shard/IndexShardModule.java +++ b/src/main/java/org/elasticsearch/index/shard/IndexShardModule.java @@ -20,17 +20,22 @@ package org.elasticsearch.index.shard; import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.shard.service.IndexShard; import org.elasticsearch.index.shard.service.InternalIndexShard; +import org.elasticsearch.jmx.JmxService; /** * */ public class IndexShardModule extends AbstractModule { + private final Settings settings; + private final ShardId shardId; - public IndexShardModule(ShardId shardId) { + public IndexShardModule(Settings settings, ShardId shardId) { + this.settings = settings; this.shardId = shardId; } @@ -38,6 +43,8 @@ public class IndexShardModule extends AbstractModule { protected void configure() { bind(ShardId.class).toInstance(shardId); bind(IndexShard.class).to(InternalIndexShard.class).asEagerSingleton(); - bind(IndexShardManagement.class).asEagerSingleton(); + if (JmxService.shouldExport(settings)) { + bind(IndexShardManagement.class).asEagerSingleton(); + } } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/index/store/StoreModule.java b/src/main/java/org/elasticsearch/index/store/StoreModule.java index ea995b74bbb..1c7865e94e4 100644 --- a/src/main/java/org/elasticsearch/index/store/StoreModule.java +++ b/src/main/java/org/elasticsearch/index/store/StoreModule.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.store; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.jmx.JmxService; /** * @@ -39,7 +40,9 @@ public class StoreModule extends AbstractModule { @Override protected void configure() { bind(DirectoryService.class).to(indexStore.shardDirectory()).asEagerSingleton(); - bind(StoreManagement.class).asEagerSingleton(); bind(Store.class).asEagerSingleton(); + if (JmxService.shouldExport(settings)) { + bind(StoreManagement.class).asEagerSingleton(); + } } } diff --git a/src/main/java/org/elasticsearch/indices/InternalIndicesService.java b/src/main/java/org/elasticsearch/indices/InternalIndicesService.java index d15762f79b2..f026b7f9ff3 100644 --- a/src/main/java/org/elasticsearch/indices/InternalIndicesService.java +++ b/src/main/java/org/elasticsearch/indices/InternalIndicesService.java @@ -279,7 +279,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent Date: Sun, 5 Feb 2012 19:06:50 +0200 Subject: [PATCH 250/270] Network: Allow to specify sub interfaces (virtual) in network configuration, closes #1667. --- .../common/network/NetworkUtils.java | 19 ++++++++---- .../monitor/network/NetworkService.java | 29 ++++++++++++++++--- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/elasticsearch/common/network/NetworkUtils.java b/src/main/java/org/elasticsearch/common/network/NetworkUtils.java index aa2ff6e49ea..b52d5d95a1e 100644 --- a/src/main/java/org/elasticsearch/common/network/NetworkUtils.java +++ b/src/main/java/org/elasticsearch/common/network/NetworkUtils.java @@ -232,12 +232,21 @@ public abstract class NetworkUtils { } + /** + * Returns all the available interfaces, including first level sub interfaces. + */ public static List getAllAvailableInterfaces() throws SocketException { - List allInterfaces = new ArrayList(10); - NetworkInterface intf; - for (Enumeration en = NetworkInterface.getNetworkInterfaces(); en.hasMoreElements(); ) { - intf = (NetworkInterface) en.nextElement(); - allInterfaces.add(intf); + List allInterfaces = new ArrayList(); + for (Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); interfaces.hasMoreElements(); ) { + NetworkInterface intf = interfaces.nextElement(); + allInterfaces.add(interfaces.nextElement()); + + Enumeration subInterfaces = intf.getSubInterfaces(); + if (subInterfaces != null && subInterfaces.hasMoreElements()) { + while (subInterfaces.hasMoreElements()) { + allInterfaces.add(subInterfaces.nextElement()); + } + } } return allInterfaces; } diff --git a/src/main/java/org/elasticsearch/monitor/network/NetworkService.java b/src/main/java/org/elasticsearch/monitor/network/NetworkService.java index dde356ea835..cced12a1b92 100644 --- a/src/main/java/org/elasticsearch/monitor/network/NetworkService.java +++ b/src/main/java/org/elasticsearch/monitor/network/NetworkService.java @@ -57,11 +57,11 @@ public class NetworkService extends AbstractComponent { if (logger.isDebugEnabled()) { StringBuilder netDebug = new StringBuilder("net_info"); try { - Enumeration enum_ = NetworkInterface.getNetworkInterfaces(); + Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); String hostName = InetAddress.getLocalHost().getHostName(); netDebug.append("\nhost [").append(hostName).append("]\n"); - while (enum_.hasMoreElements()) { - NetworkInterface net = enum_.nextElement(); + while (interfaces.hasMoreElements()) { + NetworkInterface net = interfaces.nextElement(); netDebug.append(net.getName()).append('\t').append("display_name [").append(net.getDisplayName()).append("]\n"); Enumeration addresses = net.getInetAddresses(); @@ -73,9 +73,30 @@ public class NetworkService extends AbstractComponent { netDebug.append("\t\tmtu [").append(net.getMTU()).append("] multicast [").append(net.supportsMulticast()).append("] ptp [").append(net.isPointToPoint()) .append("] loopback [").append(net.isLoopback()).append("] up [").append(net.isUp()).append("] virtual [").append(net.isVirtual()).append("]") .append('\n'); + + Enumeration subInterfaces = net.getSubInterfaces(); + if (subInterfaces != null && subInterfaces.hasMoreElements()) { + netDebug.append("\t\t\tsub interfaces:\n"); + + while (subInterfaces.hasMoreElements()) { + + net = subInterfaces.nextElement(); + + netDebug.append("\t\t\t").append(net.getName()).append("\t").append("display_name [").append(net.getDisplayName()).append("]\n"); + addresses = net.getInetAddresses(); + netDebug.append("\t\t\t\t\taddress "); + while (addresses.hasMoreElements()) { + netDebug.append("[").append(addresses.nextElement()).append("] "); + } + netDebug.append('\n'); + netDebug.append("\t\t\t\t\tmtu [").append(net.getMTU()).append("] multicast [").append(net.supportsMulticast()).append("] ptp [").append(net.isPointToPoint()) + .append("] loopback [").append(net.isLoopback()).append("] up [").append(net.isUp()).append("] virtual [").append(net.isVirtual()).append("]") + .append('\n'); + } + } } } catch (Exception ex) { - netDebug.append("Failed to get Network Interface Info [" + ex.getMessage() + "]"); + netDebug.append("failed to get Network Interface Info [" + ex.getMessage() + "]"); } logger.debug(netDebug.toString()); } From e8a0ccc20c2bf9fe7cee6699ed85886b198718d3 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 5 Feb 2012 20:12:41 +0200 Subject: [PATCH 251/270] Thread Pool: Add a specific thread pool for bulk indexing operations, closes #1668. --- .../org/elasticsearch/action/bulk/TransportShardBulkAction.java | 2 +- src/main/java/org/elasticsearch/threadpool/ThreadPool.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 034be7bd8ba..61291c0284b 100644 --- a/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -72,7 +72,7 @@ public class TransportShardBulkAction extends TransportShardReplicationOperation @Override protected String executor() { - return ThreadPool.Names.INDEX; + return ThreadPool.Names.BULK; } @Override diff --git a/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index 7f7c4737a6f..6d96606d8aa 100644 --- a/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -58,6 +58,7 @@ public class ThreadPool extends AbstractComponent { public static final String SAME = "same"; public static final String CACHED = "cached"; public static final String INDEX = "index"; + public static final String BULK = "bulk"; public static final String SEARCH = "search"; public static final String PERCOLATE = "percolate"; public static final String MANAGEMENT = "management"; @@ -85,6 +86,7 @@ public class ThreadPool extends AbstractComponent { Map executors = Maps.newHashMap(); executors.put(Names.CACHED, build(Names.CACHED, "cached", groupSettings.get(Names.CACHED), settingsBuilder().put("keep_alive", "30s").build())); executors.put(Names.INDEX, build(Names.INDEX, "cached", groupSettings.get(Names.INDEX), ImmutableSettings.Builder.EMPTY_SETTINGS)); + executors.put(Names.BULK, build(Names.BULK, "cached", groupSettings.get(Names.BULK), ImmutableSettings.Builder.EMPTY_SETTINGS)); executors.put(Names.SEARCH, build(Names.SEARCH, "cached", groupSettings.get(Names.SEARCH), ImmutableSettings.Builder.EMPTY_SETTINGS)); executors.put(Names.PERCOLATE, build(Names.PERCOLATE, "cached", groupSettings.get(Names.PERCOLATE), ImmutableSettings.Builder.EMPTY_SETTINGS)); executors.put(Names.MANAGEMENT, build(Names.MANAGEMENT, "scaling", groupSettings.get(Names.MANAGEMENT), settingsBuilder().put("keep_alive", "5m").put("size", 20).build())); From 2a4ebca0be2baf0abf7a56e29ef1bd93f294186f Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 6 Feb 2012 00:28:31 +0200 Subject: [PATCH 252/270] move to constant fields names --- .../elasticsearch/monitor/jvm/JvmInfo.java | 53 ++++++--- .../elasticsearch/monitor/jvm/JvmStats.java | 106 ++++++++++++------ 2 files changed, 110 insertions(+), 49 deletions(-) diff --git a/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java b/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java index e721a26cefd..9aea763ff8f 100644 --- a/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java +++ b/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; import java.io.Serializable; @@ -196,29 +197,49 @@ public class JvmInfo implements Streamable, Serializable, ToXContent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("jvm"); - builder.field("pid", pid); - builder.field("version", version); - builder.field("vm_name", vmName); - builder.field("vm_version", vmVersion); - builder.field("vm_vendor", vmVendor); - builder.field("start_time", startTime); + builder.startObject(Fields.JVM); + builder.field(Fields.PID, pid); + builder.field(Fields.VERSION, version); + builder.field(Fields.VM_NAME, vmName); + builder.field(Fields.VM_VERSION, vmVersion); + builder.field(Fields.VM_VENDOR, vmVendor); + builder.field(Fields.START_TIME, startTime); - builder.startObject("mem"); - builder.field("heap_init", mem.heapInit().toString()); - builder.field("heap_init_in_bytes", mem.heapInit); - builder.field("heap_max", mem.heapMax().toString()); - builder.field("heap_max_in_bytes", mem.heapMax); - builder.field("non_heap_init", mem.nonHeapInit().toString()); - builder.field("non_heap_init_in_bytes", mem.nonHeapInit); - builder.field("non_heap_max", mem.nonHeapMax().toString()); - builder.field("non_heap_max_in_bytes", mem.nonHeapMax); + builder.startObject(Fields.MEM); + builder.field(Fields.HEAP_INIT, mem.heapInit().toString()); + builder.field(Fields.HEAP_INIT_IN_BYTES, mem.heapInit); + builder.field(Fields.HEAP_MAX, mem.heapMax().toString()); + builder.field(Fields.HEAP_MAX_IN_BYTES, mem.heapMax); + builder.field(Fields.NON_HEAP_INIT, mem.nonHeapInit().toString()); + builder.field(Fields.NON_HEAP_INIT_IN_BYTES, mem.nonHeapInit); + builder.field(Fields.NON_HEAP_MAX, mem.nonHeapMax().toString()); + builder.field(Fields.NON_HEAP_MAX_IN_BYTES, mem.nonHeapMax); builder.endObject(); builder.endObject(); return builder; } + static final class Fields { + static final XContentBuilderString JVM = new XContentBuilderString("jvm"); + static final XContentBuilderString PID = new XContentBuilderString("pid"); + static final XContentBuilderString VERSION = new XContentBuilderString("version"); + static final XContentBuilderString VM_NAME = new XContentBuilderString("vm_name"); + static final XContentBuilderString VM_VERSION = new XContentBuilderString("vm_version"); + static final XContentBuilderString VM_VENDOR = new XContentBuilderString("vm_vendor"); + static final XContentBuilderString START_TIME = new XContentBuilderString("start_time"); + + static final XContentBuilderString MEM = new XContentBuilderString("mem"); + static final XContentBuilderString HEAP_INIT = new XContentBuilderString("heap_init"); + static final XContentBuilderString HEAP_INIT_IN_BYTES = new XContentBuilderString("heap_init_in_bytes"); + static final XContentBuilderString HEAP_MAX = new XContentBuilderString("heap_max"); + static final XContentBuilderString HEAP_MAX_IN_BYTES = new XContentBuilderString("heap_max_in_bytes"); + static final XContentBuilderString NON_HEAP_INIT = new XContentBuilderString("non_heap_init"); + static final XContentBuilderString NON_HEAP_INIT_IN_BYTES = new XContentBuilderString("non_heap_init_in_bytes"); + static final XContentBuilderString NON_HEAP_MAX = new XContentBuilderString("non_heap_max"); + static final XContentBuilderString NON_HEAP_MAX_IN_BYTES = new XContentBuilderString("non_heap_max_in_bytes"); + } + public static JvmInfo readJvmInfo(StreamInput in) throws IOException { JvmInfo jvmInfo = new JvmInfo(); jvmInfo.readFrom(in); diff --git a/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java b/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java index c7b348081f2..cd25a47f8b6 100644 --- a/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java +++ b/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; import java.io.Serializable; @@ -223,34 +224,34 @@ public class JvmStats implements Streamable, Serializable, ToXContent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("jvm"); - builder.field("timestamp", timestamp); - builder.field("uptime", uptime().format()); - builder.field("uptime_in_millis", uptime().millis()); + builder.startObject(Fields.JVM); + builder.field(Fields.TIMESTAMP, timestamp); + builder.field(Fields.UPTIME, uptime().format()); + builder.field(Fields.UPTIME_IN_MILLIS, uptime().millis()); if (mem != null) { - builder.startObject("mem"); - builder.field("heap_used", mem.heapUsed().toString()); - builder.field("heap_used_in_bytes", mem.heapUsed().bytes()); - builder.field("heap_committed", mem.heapCommitted().toString()); - builder.field("heap_committed_in_bytes", mem.heapCommitted().bytes()); + builder.startObject(Fields.MEM); + builder.field(Fields.HEAP_USED, mem.heapUsed().toString()); + builder.field(Fields.HEAP_USED_IN_BYTES, mem.heapUsed().bytes()); + builder.field(Fields.HEAP_COMMITTED, mem.heapCommitted().toString()); + builder.field(Fields.HEAP_COMMITTED_IN_BYTES, mem.heapCommitted().bytes()); - builder.field("non_heap_used", mem.nonHeapUsed().toString()); - builder.field("non_heap_used_in_bytes", mem.nonHeapUsed); - builder.field("non_heap_committed", mem.nonHeapCommitted().toString()); - builder.field("non_heap_committed_in_bytes", mem.nonHeapCommitted); + builder.field(Fields.NON_HEAP_USED, mem.nonHeapUsed().toString()); + builder.field(Fields.NON_HEAP_USED_IN_BYTES, mem.nonHeapUsed); + builder.field(Fields.NON_HEAP_COMMITTED, mem.nonHeapCommitted().toString()); + builder.field(Fields.NON_HEAP_COMMITTED_IN_BYTES, mem.nonHeapCommitted); - builder.startObject("pools"); + builder.startObject(Fields.POOLS); for (MemoryPool pool : mem) { builder.startObject(pool.name()); - builder.field("used", pool.used().toString()); - builder.field("used_in_bytes", pool.used); - builder.field("max", pool.max().toString()); - builder.field("max_in_bytes", pool.max); + builder.field(Fields.USED, pool.used().toString()); + builder.field(Fields.USED_IN_BYTES, pool.used); + builder.field(Fields.MAX, pool.max().toString()); + builder.field(Fields.MAX_IN_BYTES, pool.max); - builder.field("peak_used", pool.peakUsed().toString()); - builder.field("peak_used_in_bytes", pool.peakUsed); - builder.field("peak_max", pool.peakMax().toString()); - builder.field("peak_max_in_bytes", pool.peakMax); + builder.field(Fields.PEAK_USED, pool.peakUsed().toString()); + builder.field(Fields.PEAK_USED_IN_BYTES, pool.peakUsed); + builder.field(Fields.PEAK_MAX, pool.peakMax().toString()); + builder.field(Fields.PEAK_MAX_IN_BYTES, pool.peakMax); builder.endObject(); } @@ -259,23 +260,23 @@ public class JvmStats implements Streamable, Serializable, ToXContent { builder.endObject(); } if (threads != null) { - builder.startObject("threads"); - builder.field("count", threads.count()); - builder.field("peak_count", threads.peakCount()); + builder.startObject(Fields.THREADS); + builder.field(Fields.COUNT, threads.count()); + builder.field(Fields.PEAK_COUNT, threads.peakCount()); builder.endObject(); } if (gc != null) { - builder.startObject("gc"); - builder.field("collection_count", gc.collectionCount()); - builder.field("collection_time", gc.collectionTime().format()); - builder.field("collection_time_in_millis", gc.collectionTime().millis()); + builder.startObject(Fields.GC); + builder.field(Fields.COLLECTION_COUNT, gc.collectionCount()); + builder.field(Fields.COLLECTION_TIME, gc.collectionTime().format()); + builder.field(Fields.COLLECTION_TIME_IN_MILLIS, gc.collectionTime().millis()); - builder.startObject("collectors"); + builder.startObject(Fields.COLLECTORS); for (GarbageCollector collector : gc) { builder.startObject(collector.name()); - builder.field("collection_count", collector.collectionCount()); - builder.field("collection_time", collector.collectionTime().format()); - builder.field("collection_time_in_millis", collector.collectionTime().millis()); + builder.field(Fields.COLLECTION_COUNT, collector.collectionCount()); + builder.field(Fields.COLLECTION_TIME, collector.collectionTime().format()); + builder.field(Fields.COLLECTION_TIME_IN_MILLIS, collector.collectionTime().millis()); builder.endObject(); } builder.endObject(); @@ -286,6 +287,45 @@ public class JvmStats implements Streamable, Serializable, ToXContent { return builder; } + static final class Fields { + static final XContentBuilderString JVM = new XContentBuilderString("jvm"); + static final XContentBuilderString TIMESTAMP = new XContentBuilderString("timestamp"); + static final XContentBuilderString UPTIME = new XContentBuilderString("uptime"); + static final XContentBuilderString UPTIME_IN_MILLIS = new XContentBuilderString("uptime_in_millis"); + + static final XContentBuilderString MEM = new XContentBuilderString("mem"); + static final XContentBuilderString HEAP_USED = new XContentBuilderString("heap_used"); + static final XContentBuilderString HEAP_USED_IN_BYTES = new XContentBuilderString("heap_used_in_bytes"); + static final XContentBuilderString HEAP_COMMITTED = new XContentBuilderString("heap_committed"); + static final XContentBuilderString HEAP_COMMITTED_IN_BYTES = new XContentBuilderString("heap_committed_in_bytes"); + + static final XContentBuilderString NON_HEAP_USED = new XContentBuilderString("non_heap_used"); + static final XContentBuilderString NON_HEAP_USED_IN_BYTES = new XContentBuilderString("non_heap_used_in_bytes"); + static final XContentBuilderString NON_HEAP_COMMITTED = new XContentBuilderString("non_heap_committed"); + static final XContentBuilderString NON_HEAP_COMMITTED_IN_BYTES = new XContentBuilderString("non_heap_committed_in_bytes"); + + static final XContentBuilderString POOLS = new XContentBuilderString("pools"); + static final XContentBuilderString USED = new XContentBuilderString("used"); + static final XContentBuilderString USED_IN_BYTES = new XContentBuilderString("used_in_bytes"); + static final XContentBuilderString MAX = new XContentBuilderString("max"); + static final XContentBuilderString MAX_IN_BYTES = new XContentBuilderString("max_in_bytes"); + static final XContentBuilderString PEAK_USED = new XContentBuilderString("peak_used"); + static final XContentBuilderString PEAK_USED_IN_BYTES = new XContentBuilderString("peak_used_in_bytes"); + static final XContentBuilderString PEAK_MAX = new XContentBuilderString("peak_max"); + static final XContentBuilderString PEAK_MAX_IN_BYTES = new XContentBuilderString("peak_max_in_bytes"); + + static final XContentBuilderString THREADS = new XContentBuilderString("threads"); + static final XContentBuilderString COUNT = new XContentBuilderString("count"); + static final XContentBuilderString PEAK_COUNT = new XContentBuilderString("peak_count"); + + static final XContentBuilderString GC = new XContentBuilderString("gc"); + static final XContentBuilderString COLLECTORS = new XContentBuilderString("collectors"); + static final XContentBuilderString COLLECTION_COUNT = new XContentBuilderString("collection_count"); + static final XContentBuilderString COLLECTION_TIME = new XContentBuilderString("collection_time"); + static final XContentBuilderString COLLECTION_TIME_IN_MILLIS = new XContentBuilderString("collection_time_in_millis"); + } + + public static JvmStats readJvmStats(StreamInput in) throws IOException { JvmStats jvmStats = new JvmStats(); jvmStats.readFrom(in); From 7df2819549db093f50df751868a7006996e91087 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 6 Feb 2012 00:49:46 +0200 Subject: [PATCH 253/270] move to constant fields names --- .../org/elasticsearch/monitor/os/OsInfo.java | 54 +++++++++---- .../org/elasticsearch/monitor/os/OsStats.java | 79 +++++++++++++------ 2 files changed, 91 insertions(+), 42 deletions(-) diff --git a/src/main/java/org/elasticsearch/monitor/os/OsInfo.java b/src/main/java/org/elasticsearch/monitor/os/OsInfo.java index 53e1e7b82a4..8ba813fa93e 100644 --- a/src/main/java/org/elasticsearch/monitor/os/OsInfo.java +++ b/src/main/java/org/elasticsearch/monitor/os/OsInfo.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; import java.io.Serializable; @@ -77,32 +78,51 @@ public class OsInfo implements Streamable, Serializable, ToXContent { return swap(); } + static final class Fields { + static final XContentBuilderString OS = new XContentBuilderString("os"); + static final XContentBuilderString REFRESH_INTERVAL = new XContentBuilderString("refresh_interval"); + static final XContentBuilderString CPU = new XContentBuilderString("cpu"); + static final XContentBuilderString VENDOR = new XContentBuilderString("vendor"); + static final XContentBuilderString MODEL = new XContentBuilderString("model"); + static final XContentBuilderString MHZ = new XContentBuilderString("mhz"); + static final XContentBuilderString TOTAL_CORES = new XContentBuilderString("total_cores"); + static final XContentBuilderString TOTAL_SOCKETS = new XContentBuilderString("total_sockets"); + static final XContentBuilderString CORES_PER_SOCKET = new XContentBuilderString("cores_per_socket"); + static final XContentBuilderString CACHE_SIZE = new XContentBuilderString("cache_size"); + static final XContentBuilderString CACHE_SIZE_IN_BYTES = new XContentBuilderString("cache_size_in_bytes"); + + static final XContentBuilderString MEM = new XContentBuilderString("mem"); + static final XContentBuilderString SWAP = new XContentBuilderString("swap"); + static final XContentBuilderString TOTAL = new XContentBuilderString("total"); + static final XContentBuilderString TOTAL_IN_BYTES = new XContentBuilderString("total_in_bytes"); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("os"); - builder.field("refresh_interval", refreshInterval); + builder.startObject(Fields.OS); + builder.field(Fields.REFRESH_INTERVAL, refreshInterval); if (cpu != null) { - builder.startObject("cpu"); - builder.field("vendor", cpu.vendor()); - builder.field("model", cpu.model()); - builder.field("mhz", cpu.mhz()); - builder.field("total_cores", cpu.totalCores()); - builder.field("total_sockets", cpu.totalSockets()); - builder.field("cores_per_socket", cpu.coresPerSocket()); - builder.field("cache_size", cpu.cacheSize().toString()); - builder.field("cache_size_in_bytes", cpu.cacheSize().bytes()); + builder.startObject(Fields.CPU); + builder.field(Fields.VENDOR, cpu.vendor()); + builder.field(Fields.MODEL, cpu.model()); + builder.field(Fields.MHZ, cpu.mhz()); + builder.field(Fields.TOTAL_CORES, cpu.totalCores()); + builder.field(Fields.TOTAL_SOCKETS, cpu.totalSockets()); + builder.field(Fields.CORES_PER_SOCKET, cpu.coresPerSocket()); + builder.field(Fields.CACHE_SIZE, cpu.cacheSize().toString()); + builder.field(Fields.CACHE_SIZE_IN_BYTES, cpu.cacheSize().bytes()); builder.endObject(); } if (mem != null) { - builder.startObject("mem"); - builder.field("total", mem.total().toString()); - builder.field("total_in_bytes", mem.total().bytes()); + builder.startObject(Fields.MEM); + builder.field(Fields.TOTAL, mem.total().toString()); + builder.field(Fields.TOTAL_IN_BYTES, mem.total); builder.endObject(); } if (swap != null) { - builder.startObject("swap"); - builder.field("total", swap.total().toString()); - builder.field("total_in_bytes", swap.total().bytes()); + builder.startObject(Fields.SWAP); + builder.field(Fields.TOTAL, swap.total().toString()); + builder.field(Fields.TOTAL_IN_BYTES, swap.total); builder.endObject(); } builder.endObject(); diff --git a/src/main/java/org/elasticsearch/monitor/os/OsStats.java b/src/main/java/org/elasticsearch/monitor/os/OsStats.java index e9bc6410a41..566f97dbc5e 100644 --- a/src/main/java/org/elasticsearch/monitor/os/OsStats.java +++ b/src/main/java/org/elasticsearch/monitor/os/OsStats.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; import java.io.Serializable; @@ -102,52 +103,80 @@ public class OsStats implements Streamable, Serializable, ToXContent { return swap(); } + static final class Fields { + static final XContentBuilderString OS = new XContentBuilderString("os"); + static final XContentBuilderString TIMESTAMP = new XContentBuilderString("timestamp"); + static final XContentBuilderString UPTIME = new XContentBuilderString("uptime"); + static final XContentBuilderString UPTIME_IN_MILLIS = new XContentBuilderString("uptime_in_millis"); + static final XContentBuilderString LOAD_AVERAGE = new XContentBuilderString("load_average"); + + static final XContentBuilderString CPU = new XContentBuilderString("cpu"); + static final XContentBuilderString SYS = new XContentBuilderString("sys"); + static final XContentBuilderString USER = new XContentBuilderString("user"); + static final XContentBuilderString IDLE = new XContentBuilderString("idle"); + + static final XContentBuilderString MEM = new XContentBuilderString("mem"); + static final XContentBuilderString SWAP = new XContentBuilderString("swap"); + static final XContentBuilderString FREE = new XContentBuilderString("free"); + static final XContentBuilderString FREE_IN_BYTES = new XContentBuilderString("free_in_bytes"); + static final XContentBuilderString USED = new XContentBuilderString("free"); + static final XContentBuilderString USED_IN_BYTES = new XContentBuilderString("free_in_bytes"); + + static final XContentBuilderString FREE_PERCENT = new XContentBuilderString("free_percent"); + static final XContentBuilderString USED_PERCENT = new XContentBuilderString("used_percent"); + + static final XContentBuilderString ACTUAL_FREE = new XContentBuilderString("actual_free"); + static final XContentBuilderString ACTUAL_FREE_IN_BYTES = new XContentBuilderString("actual_free_in_bytes"); + static final XContentBuilderString ACTUAL_USED = new XContentBuilderString("actual_used"); + static final XContentBuilderString ACTUAL_USED_IN_BYTES = new XContentBuilderString("actual_used_in_bytes"); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("os"); - builder.field("timestamp", timestamp); + builder.startObject(Fields.OS); + builder.field(Fields.TIMESTAMP, timestamp); - builder.field("uptime", uptime().format()); - builder.field("uptime_in_millis", uptime().millis()); + builder.field(Fields.UPTIME, uptime().format()); + builder.field(Fields.UPTIME_IN_MILLIS, uptime().millis()); - builder.startArray("load_average"); + builder.startArray(Fields.LOAD_AVERAGE); for (double value : loadAverage) { builder.value(value); } builder.endArray(); if (cpu != null) { - builder.startObject("cpu"); - builder.field("sys", cpu.sys()); - builder.field("user", cpu.user()); - builder.field("idle", cpu.idle()); + builder.startObject(Fields.CPU); + builder.field(Fields.SYS, cpu.sys()); + builder.field(Fields.USER, cpu.user()); + builder.field(Fields.IDLE, cpu.idle()); builder.endObject(); } if (mem != null) { - builder.startObject("mem"); - builder.field("free", mem.free().toString()); - builder.field("free_in_bytes", mem.free().bytes()); - builder.field("used", mem.used().toString()); - builder.field("used_in_bytes", mem.used().bytes()); + builder.startObject(Fields.MEM); + builder.field(Fields.FREE, mem.free().toString()); + builder.field(Fields.FREE_IN_BYTES, mem.free); + builder.field(Fields.USED, mem.used().toString()); + builder.field(Fields.USED_IN_BYTES, mem.used); - builder.field("free_percent", mem.freePercent()); - builder.field("used_percent", mem.usedPercent()); + builder.field(Fields.FREE_PERCENT, mem.freePercent()); + builder.field(Fields.USED_PERCENT, mem.usedPercent()); - builder.field("actual_free", mem.actualFree().toString()); - builder.field("actual_free_in_bytes", mem.actualFree().bytes()); - builder.field("actual_used", mem.actualUsed().toString()); - builder.field("actual_used_in_bytes", mem.actualUsed().bytes()); + builder.field(Fields.ACTUAL_FREE, mem.actualFree().toString()); + builder.field(Fields.ACTUAL_FREE_IN_BYTES, mem.actualFree); + builder.field(Fields.ACTUAL_USED, mem.actualUsed().toString()); + builder.field(Fields.ACTUAL_USED_IN_BYTES, mem.actualUsed); builder.endObject(); } if (swap != null) { - builder.startObject("swap"); - builder.field("used", swap.used().toString()); - builder.field("used_in_bytes", swap.used().bytes()); - builder.field("free", swap.free().toString()); - builder.field("free_in_bytes", swap.free().bytes()); + builder.startObject(Fields.SWAP); + builder.field(Fields.USED, swap.used().toString()); + builder.field(Fields.USED_IN_BYTES, swap.used); + builder.field(Fields.FREE, swap.free().toString()); + builder.field(Fields.FREE_IN_BYTES, swap.free); builder.endObject(); } From 5c4c319a4155c69ca877e578b80895009b75d876 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 6 Feb 2012 03:33:22 +0200 Subject: [PATCH 254/270] move to constant fields names --- .../java/org/elasticsearch/http/HttpInfo.java | 13 +++- .../org/elasticsearch/http/HttpStats.java | 13 +++- .../org/elasticsearch/monitor/fs/FsStats.java | 64 +++++++++++++------ .../monitor/network/NetworkInfo.java | 22 +++++-- .../monitor/network/NetworkStats.java | 40 ++++++++---- .../monitor/process/ProcessInfo.java | 16 +++-- .../monitor/process/ProcessStats.java | 60 +++++++++++------ .../elasticsearch/threadpool/ThreadPool.java | 20 ++++-- .../threadpool/ThreadPoolInfo.java | 7 +- .../threadpool/ThreadPoolStats.java | 13 +++- .../transport/TransportInfo.java | 13 +++- .../transport/TransportStats.java | 28 +++++--- 12 files changed, 223 insertions(+), 86 deletions(-) diff --git a/src/main/java/org/elasticsearch/http/HttpInfo.java b/src/main/java/org/elasticsearch/http/HttpInfo.java index 26631d86f6a..ee6af230143 100644 --- a/src/main/java/org/elasticsearch/http/HttpInfo.java +++ b/src/main/java/org/elasticsearch/http/HttpInfo.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; import java.io.Serializable; @@ -43,11 +44,17 @@ public class HttpInfo implements Streamable, Serializable, ToXContent { this.address = address; } + static final class Fields { + static final XContentBuilderString HTTP = new XContentBuilderString("http"); + static final XContentBuilderString BOUND_ADDRESS = new XContentBuilderString("bound_address"); + static final XContentBuilderString PUBLISH_ADDRESS = new XContentBuilderString("publish_address"); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("http"); - builder.field("bound_address", address.boundAddress().toString()); - builder.field("publish_address", address.publishAddress().toString()); + builder.startObject(Fields.HTTP); + builder.field(Fields.BOUND_ADDRESS, address.boundAddress().toString()); + builder.field(Fields.PUBLISH_ADDRESS, address.publishAddress().toString()); builder.endObject(); return builder; } diff --git a/src/main/java/org/elasticsearch/http/HttpStats.java b/src/main/java/org/elasticsearch/http/HttpStats.java index 6a67e552c85..6b7cfa62983 100644 --- a/src/main/java/org/elasticsearch/http/HttpStats.java +++ b/src/main/java/org/elasticsearch/http/HttpStats.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; @@ -75,11 +76,17 @@ public class HttpStats implements Streamable, ToXContent { out.writeVLong(totalOpen); } + static final class Fields { + static final XContentBuilderString HTTP = new XContentBuilderString("http"); + static final XContentBuilderString CURRENT_OPEN = new XContentBuilderString("current_open"); + static final XContentBuilderString TOTAL_OPENED = new XContentBuilderString("total_opened"); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("http"); - builder.field("current_open", serverOpen); - builder.field("total_opened", totalOpen); + builder.startObject(Fields.HTTP); + builder.field(Fields.CURRENT_OPEN, serverOpen); + builder.field(Fields.TOTAL_OPENED, totalOpen); builder.endObject(); return builder; } diff --git a/src/main/java/org/elasticsearch/monitor/fs/FsStats.java b/src/main/java/org/elasticsearch/monitor/fs/FsStats.java index f18749177c1..7697d08ea33 100644 --- a/src/main/java/org/elasticsearch/monitor/fs/FsStats.java +++ b/src/main/java/org/elasticsearch/monitor/fs/FsStats.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; import java.util.Iterator; @@ -224,55 +225,78 @@ public class FsStats implements Iterable, Streamable, ToXContent { } } + static final class Fields { + static final XContentBuilderString FS = new XContentBuilderString("fs"); + static final XContentBuilderString TIMESTAMP = new XContentBuilderString("timestamp"); + static final XContentBuilderString DATA = new XContentBuilderString("data"); + static final XContentBuilderString PATH = new XContentBuilderString("path"); + static final XContentBuilderString MOUNT = new XContentBuilderString("mount"); + static final XContentBuilderString DEV = new XContentBuilderString("dev"); + static final XContentBuilderString TOTAL = new XContentBuilderString("total"); + static final XContentBuilderString TOTAL_IN_BYTES = new XContentBuilderString("total_in_bytes"); + static final XContentBuilderString FREE = new XContentBuilderString("free"); + static final XContentBuilderString FREE_IN_BYTES = new XContentBuilderString("free_in_bytes"); + static final XContentBuilderString AVAILABLE = new XContentBuilderString("available"); + static final XContentBuilderString AVAILABLE_IN_BYTES = new XContentBuilderString("available_in_bytes"); + static final XContentBuilderString DISK_READS = new XContentBuilderString("disk_reads"); + static final XContentBuilderString DISK_WRITES = new XContentBuilderString("disk_writes"); + static final XContentBuilderString DISK_READ_SIZE = new XContentBuilderString("disk_read_size"); + static final XContentBuilderString DISK_READ_SIZE_IN_BYTES = new XContentBuilderString("disk_read_size_in_bytes"); + static final XContentBuilderString DISK_WRITE_SIZE = new XContentBuilderString("disk_write_size"); + static final XContentBuilderString DISK_WRITE_SIZE_IN_BYTES = new XContentBuilderString("disk_write_size_in_bytes"); + static final XContentBuilderString DISK_QUEUE = new XContentBuilderString("disk_queue"); + static final XContentBuilderString DISK_SERVICE_TIME = new XContentBuilderString("disk_service_time"); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("fs"); - builder.field("timestamp", timestamp); - builder.startArray("data"); + builder.startObject(Fields.FS); + builder.field(Fields.TIMESTAMP, timestamp); + builder.startArray(Fields.DATA); for (Info info : infos) { builder.startObject(); - builder.field("path", info.path); + builder.field(Fields.PATH, info.path); if (info.mount != null) { - builder.field("mount", info.mount); + builder.field(Fields.MOUNT, info.mount); } if (info.dev != null) { - builder.field("dev", info.dev); + builder.field(Fields.DEV, info.dev); } if (info.total != -1) { - builder.field("total", info.total().toString()); - builder.field("total_in_bytes", info.total); + builder.field(Fields.TOTAL, info.total().toString()); + builder.field(Fields.TOTAL_IN_BYTES, info.total); } if (info.free != -1) { - builder.field("free", info.free().toString()); - builder.field("free_in_bytes", info.free); + builder.field(Fields.FREE, info.free().toString()); + builder.field(Fields.FREE_IN_BYTES, info.free); } if (info.available != -1) { - builder.field("available", info.available().toString()); - builder.field("available_in_bytes", info.available); + builder.field(Fields.AVAILABLE, info.available().toString()); + builder.field(Fields.AVAILABLE_IN_BYTES, info.available); } if (info.diskReads != -1) { - builder.field("disk_reads", info.diskReads); + builder.field(Fields.DISK_READS, info.diskReads); } if (info.diskWrites != -1) { - builder.field("disk_writes", info.diskWrites); + builder.field(Fields.DISK_WRITES, info.diskWrites); } if (info.diskReadBytes != -1) { - builder.field("disk_read_size", info.diskReadSizeSize().toString()); - builder.field("disk_read_size_bytes", info.diskReadSizeInBytes()); + builder.field(Fields.DISK_READ_SIZE, info.diskReadSizeSize().toString()); + builder.field(Fields.DISK_READ_SIZE_IN_BYTES, info.diskReadSizeInBytes()); } if (info.diskWriteBytes != -1) { - builder.field("disk_write_size", info.diskWriteSizeSize().toString()); - builder.field("disk_write_size_bytes", info.diskWriteSizeInBytes()); + builder.field(Fields.DISK_WRITE_SIZE, info.diskWriteSizeSize().toString()); + builder.field(Fields.DISK_WRITE_SIZE_IN_BYTES, info.diskWriteSizeInBytes()); } if (info.diskQueue != -1) { - builder.field("disk_queue", Strings.format1Decimals(info.diskQueue, "")); + builder.field(Fields.DISK_QUEUE, Strings.format1Decimals(info.diskQueue, "")); } if (info.diskServiceTime != -1) { - builder.field("disk_service_time", Strings.format1Decimals(info.diskServiceTime, "")); + builder.field(Fields.DISK_SERVICE_TIME, Strings.format1Decimals(info.diskServiceTime, "")); } builder.endObject(); diff --git a/src/main/java/org/elasticsearch/monitor/network/NetworkInfo.java b/src/main/java/org/elasticsearch/monitor/network/NetworkInfo.java index a31791ca8ae..818a2835029 100644 --- a/src/main/java/org/elasticsearch/monitor/network/NetworkInfo.java +++ b/src/main/java/org/elasticsearch/monitor/network/NetworkInfo.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; import java.io.Serializable; @@ -55,15 +56,24 @@ public class NetworkInfo implements Streamable, Serializable, ToXContent { return primaryInterface(); } + static final class Fields { + static final XContentBuilderString NETWORK = new XContentBuilderString("network"); + static final XContentBuilderString REFRESH_INTERVAL = new XContentBuilderString("refresh_interval"); + static final XContentBuilderString PRIMARY_INTERFACE = new XContentBuilderString("primary_interface"); + static final XContentBuilderString ADDRESS = new XContentBuilderString("address"); + static final XContentBuilderString NAME = new XContentBuilderString("name"); + static final XContentBuilderString MAC_ADDRESS = new XContentBuilderString("mac_address"); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("network"); - builder.field("refresh_interval", refreshInterval); + builder.startObject(Fields.NETWORK); + builder.field(Fields.REFRESH_INTERVAL, refreshInterval); if (primary != NA_INTERFACE) { - builder.startObject("primary_interface"); - builder.field("address", primary.address()); - builder.field("name", primary.name()); - builder.field("mac_address", primary.macAddress()); + builder.startObject(Fields.PRIMARY_INTERFACE); + builder.field(Fields.ADDRESS, primary.address()); + builder.field(Fields.NAME, primary.name()); + builder.field(Fields.MAC_ADDRESS, primary.macAddress()); builder.endObject(); } builder.endObject(); diff --git a/src/main/java/org/elasticsearch/monitor/network/NetworkStats.java b/src/main/java/org/elasticsearch/monitor/network/NetworkStats.java index a071402d774..13cc9040a0f 100644 --- a/src/main/java/org/elasticsearch/monitor/network/NetworkStats.java +++ b/src/main/java/org/elasticsearch/monitor/network/NetworkStats.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; import java.io.Serializable; @@ -41,21 +42,36 @@ public class NetworkStats implements Streamable, Serializable, ToXContent { } + static final class Fields { + static final XContentBuilderString NETWORK = new XContentBuilderString("network"); + static final XContentBuilderString TCP = new XContentBuilderString("tcp"); + static final XContentBuilderString ACTIVE_OPENS = new XContentBuilderString("active_opens"); + static final XContentBuilderString PASSIVE_OPENS = new XContentBuilderString("passive_opens"); + static final XContentBuilderString CURR_ESTAB = new XContentBuilderString("curr_estab"); + static final XContentBuilderString IN_SEGS = new XContentBuilderString("in_segs"); + static final XContentBuilderString OUT_SEGS = new XContentBuilderString("out_segs"); + static final XContentBuilderString RETRANS_SEGS = new XContentBuilderString("retrans_segs"); + static final XContentBuilderString ESTAB_RESETS = new XContentBuilderString("estab_resets"); + static final XContentBuilderString ATTEMPT_FAILS = new XContentBuilderString("attempt_fails"); + static final XContentBuilderString IN_ERRS = new XContentBuilderString("in_errs"); + static final XContentBuilderString OUT_RSTS = new XContentBuilderString("out_rsts"); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("network"); + builder.startObject(Fields.NETWORK); if (tcp != null) { - builder.startObject("tcp"); - builder.field("active_opens", tcp.getActiveOpens()); - builder.field("passive_opens", tcp.getPassiveOpens()); - builder.field("curr_estab", tcp.getCurrEstab()); - builder.field("in_segs", tcp.getInSegs()); - builder.field("out_segs", tcp.getOutSegs()); - builder.field("retrans_segs", tcp.getRetransSegs()); - builder.field("estab_resets", tcp.getEstabResets()); - builder.field("attempt_fails", tcp.getAttemptFails()); - builder.field("in_errs", tcp.getInErrs()); - builder.field("out_rsts", tcp.getOutRsts()); + builder.startObject(Fields.TCP); + builder.field(Fields.ACTIVE_OPENS, tcp.getActiveOpens()); + builder.field(Fields.PASSIVE_OPENS, tcp.getPassiveOpens()); + builder.field(Fields.CURR_ESTAB, tcp.getCurrEstab()); + builder.field(Fields.IN_SEGS, tcp.getInSegs()); + builder.field(Fields.OUT_SEGS, tcp.getOutSegs()); + builder.field(Fields.RETRANS_SEGS, tcp.getRetransSegs()); + builder.field(Fields.ESTAB_RESETS, tcp.getEstabResets()); + builder.field(Fields.ATTEMPT_FAILS, tcp.getAttemptFails()); + builder.field(Fields.IN_ERRS, tcp.getInErrs()); + builder.field(Fields.OUT_RSTS, tcp.getOutRsts()); builder.endObject(); } builder.endObject(); diff --git a/src/main/java/org/elasticsearch/monitor/process/ProcessInfo.java b/src/main/java/org/elasticsearch/monitor/process/ProcessInfo.java index 2c2a582ccf3..ead00aff23b 100644 --- a/src/main/java/org/elasticsearch/monitor/process/ProcessInfo.java +++ b/src/main/java/org/elasticsearch/monitor/process/ProcessInfo.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; import java.io.Serializable; @@ -78,12 +79,19 @@ public class ProcessInfo implements Streamable, Serializable, ToXContent { return maxFileDescriptors; } + static final class Fields { + static final XContentBuilderString PROCESS = new XContentBuilderString("process"); + static final XContentBuilderString REFRESH_INTERVAL = new XContentBuilderString("refresh_interval"); + static final XContentBuilderString ID = new XContentBuilderString("id"); + static final XContentBuilderString MAX_FILE_DESCRIPTORS = new XContentBuilderString("max_file_descriptors"); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("process"); - builder.field("refresh_interval", refreshInterval); - builder.field("id", id); - builder.field("max_file_descriptors", maxFileDescriptors); + builder.startObject(Fields.PROCESS); + builder.field(Fields.REFRESH_INTERVAL, refreshInterval); + builder.field(Fields.ID, id); + builder.field(Fields.MAX_FILE_DESCRIPTORS, maxFileDescriptors); builder.endObject(); return builder; } diff --git a/src/main/java/org/elasticsearch/monitor/process/ProcessStats.java b/src/main/java/org/elasticsearch/monitor/process/ProcessStats.java index 9e0a0e98043..f8957efa430 100644 --- a/src/main/java/org/elasticsearch/monitor/process/ProcessStats.java +++ b/src/main/java/org/elasticsearch/monitor/process/ProcessStats.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; import java.io.Serializable; @@ -78,30 +79,53 @@ public class ProcessStats implements Streamable, Serializable, ToXContent { return mem(); } + static final class Fields { + static final XContentBuilderString PROCESS = new XContentBuilderString("process"); + static final XContentBuilderString TIMESTAMP = new XContentBuilderString("timestamp"); + static final XContentBuilderString OPEN_FILE_DESCRIPTORS = new XContentBuilderString("open_file_descriptors"); + + static final XContentBuilderString CPU = new XContentBuilderString("cpu"); + static final XContentBuilderString PERCENT = new XContentBuilderString("percent"); + static final XContentBuilderString SYS = new XContentBuilderString("sys"); + static final XContentBuilderString SYS_IN_MILLIS = new XContentBuilderString("sys_in_millis"); + static final XContentBuilderString USER = new XContentBuilderString("user"); + static final XContentBuilderString USER_IN_MILLIS = new XContentBuilderString("user_in_millis"); + static final XContentBuilderString TOTAL = new XContentBuilderString("total"); + static final XContentBuilderString TOTAL_IN_MILLIS = new XContentBuilderString("total_in_millis"); + + static final XContentBuilderString MEM = new XContentBuilderString("mem"); + static final XContentBuilderString RESIDENT = new XContentBuilderString("resident"); + static final XContentBuilderString RESIDENT_IN_BYTES = new XContentBuilderString("resident_in_bytes"); + static final XContentBuilderString SHARE = new XContentBuilderString("share"); + static final XContentBuilderString SHARE_IN_BYTES = new XContentBuilderString("share_in_bytes"); + static final XContentBuilderString TOTAL_VIRTUAL = new XContentBuilderString("total_virtual"); + static final XContentBuilderString TOTAL_VIRTUAL_IN_BYTES = new XContentBuilderString("total_virtual_in_bytes"); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("process"); - builder.field("timestamp", timestamp); - builder.field("open_file_descriptors", openFileDescriptors); + builder.startObject(Fields.PROCESS); + builder.field(Fields.TIMESTAMP, timestamp); + builder.field(Fields.OPEN_FILE_DESCRIPTORS, openFileDescriptors); if (cpu != null) { - builder.startObject("cpu"); - builder.field("percent", cpu.percent()); - builder.field("sys", cpu.sys().format()); - builder.field("sys_in_millis", cpu.sys().millis()); - builder.field("user", cpu.user().format()); - builder.field("user_in_millis", cpu.user().millis()); - builder.field("total", cpu.total().format()); - builder.field("total_in_millis", cpu.total().millis()); + builder.startObject(Fields.CPU); + builder.field(Fields.PERCENT, cpu.percent()); + builder.field(Fields.SYS, cpu.sys().format()); + builder.field(Fields.SYS_IN_MILLIS, cpu.sys().millis()); + builder.field(Fields.USER, cpu.user().format()); + builder.field(Fields.USER_IN_MILLIS, cpu.user().millis()); + builder.field(Fields.TOTAL, cpu.total().format()); + builder.field(Fields.TOTAL_IN_MILLIS, cpu.total().millis()); builder.endObject(); } if (mem != null) { - builder.startObject("mem"); - builder.field("resident", mem.resident().toString()); - builder.field("resident_in_bytes", mem.resident().bytes()); - builder.field("share", mem.share().toString()); - builder.field("share_in_bytes", mem.share().bytes()); - builder.field("total_virtual", mem.totalVirtual().toString()); - builder.field("total_virtual_in_bytes", mem.totalVirtual().bytes()); + builder.startObject(Fields.MEM); + builder.field(Fields.RESIDENT, mem.resident().toString()); + builder.field(Fields.RESIDENT_IN_BYTES, mem.resident().bytes()); + builder.field(Fields.SHARE, mem.share().toString()); + builder.field(Fields.SHARE_IN_BYTES, mem.share().bytes()); + builder.field(Fields.TOTAL_VIRTUAL, mem.totalVirtual().toString()); + builder.field(Fields.TOTAL_VIRTUAL_IN_BYTES, mem.totalVirtual().bytes()); builder.endObject(); } builder.endObject(); diff --git a/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index 6d96606d8aa..0f3c57a99e7 100644 --- a/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; import java.util.ArrayList; @@ -504,21 +505,30 @@ public class ThreadPool extends AbstractComponent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(name, XContentBuilder.FieldCaseConversion.NONE); - builder.field("type", type); + builder.field(Fields.TYPE, type); if (min != -1) { - builder.field("min", min); + builder.field(Fields.MIN, min); } if (max != -1) { - builder.field("max", max); + builder.field(Fields.MAX, max); } if (keepAlive != null) { - builder.field("keep_alive", keepAlive.toString()); + builder.field(Fields.KEEP_ALIVE, keepAlive.toString()); } if (capacity != null) { - builder.field("capacity", capacity.toString()); + builder.field(Fields.CAPACITY, capacity.toString()); } builder.endObject(); return builder; } + + static final class Fields { + static final XContentBuilderString TYPE = new XContentBuilderString("type"); + static final XContentBuilderString MIN = new XContentBuilderString("min"); + static final XContentBuilderString MAX = new XContentBuilderString("max"); + static final XContentBuilderString KEEP_ALIVE = new XContentBuilderString("keep_alive"); + static final XContentBuilderString CAPACITY = new XContentBuilderString("capacity"); + } + } } diff --git a/src/main/java/org/elasticsearch/threadpool/ThreadPoolInfo.java b/src/main/java/org/elasticsearch/threadpool/ThreadPoolInfo.java index dda8b25d4a0..5eeab7d53a9 100644 --- a/src/main/java/org/elasticsearch/threadpool/ThreadPoolInfo.java +++ b/src/main/java/org/elasticsearch/threadpool/ThreadPoolInfo.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; import java.util.ArrayList; @@ -74,9 +75,13 @@ public class ThreadPoolInfo implements Streamable, Iterable, To } } + static final class Fields { + static final XContentBuilderString THREAD_POOL = new XContentBuilderString("thread_pool"); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("thread_pool"); + builder.startObject(Fields.THREAD_POOL); for (ThreadPool.Info info : infos) { info.toXContent(builder, params); } diff --git a/src/main/java/org/elasticsearch/threadpool/ThreadPoolStats.java b/src/main/java/org/elasticsearch/threadpool/ThreadPoolStats.java index a9476cfd1cd..564a0ba2847 100644 --- a/src/main/java/org/elasticsearch/threadpool/ThreadPoolStats.java +++ b/src/main/java/org/elasticsearch/threadpool/ThreadPoolStats.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; import java.util.ArrayList; @@ -92,10 +93,10 @@ public class ThreadPoolStats implements Streamable, ToXContent, Iterable Date: Mon, 6 Feb 2012 04:08:26 +0200 Subject: [PATCH 255/270] upgrade to mvel 2.1.Beta8 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3d18112dcfc..d828aab5fed 100644 --- a/pom.xml +++ b/pom.xml @@ -125,7 +125,7 @@ org.mvel mvel2 - 2.1.Beta7 + 2.1.Beta8 compile From f3a5306cde7ab71958492421261fa53d08eb1732 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 6 Feb 2012 21:03:12 +0200 Subject: [PATCH 256/270] improve comment on usage, also, FrameDecoder usage of cumalation was fixed --- .../elasticsearch/transport/netty/MessageChannelHandler.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java b/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java index bd2f1afb8e6..18827611620 100644 --- a/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java +++ b/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java @@ -68,8 +68,9 @@ public class MessageChannelHandler extends SimpleChannelUpstreamHandler { // similar logic to FrameDecoder, we don't use FrameDecoder because we can use the data len header value // to guess the size of the cumulation buffer to allocate - // Also strange, is that the FrameDecoder always allocated a cumulation, even if the input bufer is enough - // so we don't allocate a cumulation buffer unless we really need to here (need to post this to the mailing list) + + // we don't reuse the cumalation buffer, so it won't grow out of control per channel, as well as + // being able to "readBytesReference" from it without worry @Override public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { From abfc57076201b9976bc51c336a8ec161cf8de7c9 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 6 Feb 2012 21:24:37 +0200 Subject: [PATCH 257/270] better comment --- .../java/org/elasticsearch/http/netty/NettyHttpRequest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java b/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java index 682d37c5b28..e2defd0faa0 100644 --- a/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java +++ b/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java @@ -110,6 +110,9 @@ public class NettyHttpRequest extends AbstractRestRequest implements HttpRequest public boolean contentUnsafe() { // the netty HTTP handling always copy over the buffer to its own buffer, either in NioWorker internally // when reading, or using a cumalation buffer + + // also, HttpMessageDecoder#content variable gets freshly created for each request and not reused across + // requests return false; //return request.getContent().hasArray(); } From 43809211d41c1ea0ee73fd5d49fb41003149d919 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 6 Feb 2012 22:09:32 +0200 Subject: [PATCH 258/270] bin: elasticsearch script to support ES_HEAP_SIZE to easily set the heap size to a single value (min and max) and ES_HEAP_NEWSIZE to optionally set the new gen, closes #1671. --- bin/elasticsearch.bat | 10 ++++++++++ bin/elasticsearch.in.sh | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/bin/elasticsearch.bat b/bin/elasticsearch.bat index 00388e7d0aa..dfa4e7c093d 100644 --- a/bin/elasticsearch.bat +++ b/bin/elasticsearch.bat @@ -18,7 +18,17 @@ if "%ES_MAX_MEM%" == "" ( set ES_MAX_MEM=1g ) +if NOT "%ES_HEAP_SIZE%" == "" ( +set ES_MIN_MEM=%ES_HEAP_SIZE% +set ES_MAX_MEM=%ES_HEAP_SIZE% +) + set JAVA_OPTS=%JAVA_OPTS% -Xms%ES_MIN_MEM% -Xmx%ES_MAX_MEM% + +if NOT "%ES_HEAP_NEWSIZE%" == "" ( +set JAVA_OPTS=%JAVA_OPTS% -Xmn%ES_HEAP_NEWSIZE% +) + set JAVA_OPTS=%JAVA_OPTS% -Xss128k REM Enable aggressive optimizations in the JVM diff --git a/bin/elasticsearch.in.sh b/bin/elasticsearch.in.sh index 7ee2c9b039f..c923d9c1424 100644 --- a/bin/elasticsearch.in.sh +++ b/bin/elasticsearch.in.sh @@ -6,6 +6,10 @@ fi if [ "x$ES_MAX_MEM" = "x" ]; then ES_MAX_MEM=1g fi +if [ "x$ES_HEAP_SIZE" != "x" ]; then + ES_MIN_MEM=$ES_HEAP_SIZE + ES_MAX_MEM=$ES_HEAP_SIZE +fi # min and max heap sizes should be set to the same value to avoid # stop-the-world GC pauses during resize, and so that we can lock the @@ -14,6 +18,11 @@ fi JAVA_OPTS="$JAVA_OPTS -Xms${ES_MIN_MEM}" JAVA_OPTS="$JAVA_OPTS -Xmx${ES_MAX_MEM}" +# new generation +if [ "x$ES_HEAP_NEWSIZE" != "x" ]; then + JAVA_OPTS="$JAVA_OPTS -Xmn${ES_HEAP_NEWSIZE}" +fi + # reduce the per-thread stack size JAVA_OPTS="$JAVA_OPTS -Xss128k" From 70de95373cc30be4a4cff8b0e6fc1ed50788be08 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 6 Feb 2012 22:11:22 +0200 Subject: [PATCH 259/270] JVM Flags: Remove SurvivorRatio and MaxTenuringThreshold since the defaults are good with new JVMs, closes #1672. --- bin/elasticsearch.bat | 4 +--- bin/elasticsearch.in.sh | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/bin/elasticsearch.bat b/bin/elasticsearch.bat index dfa4e7c093d..14c7ff989da 100644 --- a/bin/elasticsearch.bat +++ b/bin/elasticsearch.bat @@ -37,9 +37,7 @@ REM set JAVA_OPTS=%JAVA_OPTS% -XX:+AggressiveOpts set JAVA_OPTS=%JAVA_OPTS% -XX:+UseParNewGC set JAVA_OPTS=%JAVA_OPTS% -XX:+UseConcMarkSweepGC -set JAVA_OPTS=%JAVA_OPTS% -XX:+CMSParallelRemarkEnabled -set JAVA_OPTS=%JAVA_OPTS% -XX:SurvivorRatio=8 -set JAVA_OPTS=%JAVA_OPTS% -XX:MaxTenuringThreshold=1 + set JAVA_OPTS=%JAVA_OPTS% -XX:CMSInitiatingOccupancyFraction=75 set JAVA_OPTS=%JAVA_OPTS% -XX:+UseCMSInitiatingOccupancyOnly diff --git a/bin/elasticsearch.in.sh b/bin/elasticsearch.in.sh index c923d9c1424..aaeb857c168 100644 --- a/bin/elasticsearch.in.sh +++ b/bin/elasticsearch.in.sh @@ -35,9 +35,7 @@ JAVA_OPTS="$JAVA_OPTS -Xss128k" JAVA_OPTS="$JAVA_OPTS -XX:+UseParNewGC" JAVA_OPTS="$JAVA_OPTS -XX:+UseConcMarkSweepGC" -JAVA_OPTS="$JAVA_OPTS -XX:+CMSParallelRemarkEnabled" -JAVA_OPTS="$JAVA_OPTS -XX:SurvivorRatio=8" -JAVA_OPTS="$JAVA_OPTS -XX:MaxTenuringThreshold=1" + JAVA_OPTS="$JAVA_OPTS -XX:CMSInitiatingOccupancyFraction=75" JAVA_OPTS="$JAVA_OPTS -XX:+UseCMSInitiatingOccupancyOnly" From aee0e1dfb776ef336906dd3695f5dc7f159bf321 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 6 Feb 2012 22:13:15 +0200 Subject: [PATCH 260/270] doc the ES_HEAP_SIZE --- bin/elasticsearch | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/elasticsearch b/bin/elasticsearch index c91587ee289..4a716a40519 100755 --- a/bin/elasticsearch +++ b/bin/elasticsearch @@ -17,8 +17,7 @@ # Optionally, exact memory values can be set using the following values, note, # they can still be set using the `ES_JAVA_OPTS`. Sample format include "512m", and "10g". # -# ES_MIN_MEM -- The minimum number of memory to allocate. -# ES_MAX_MEM -- The maximum number of memory to allocate. +# ES_HEAP_SIZE -- Sets both the minimum and maximum memory to allocate (recommended) # # As a convenience, a fragment of shell is sourced in order to set one or # more of these variables. This so-called `include' can be placed in a From 768a23e0cbd6d5beac502ee235ea88b0b26a497c Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 7 Feb 2012 00:37:59 +0200 Subject: [PATCH 261/270] Node Stats: Add more options to get specific stats in REST: /_nodes/fs/stats (in addition to /_nodes/stats/fs), closes #1674. --- .../node/stats/RestNodesStatsAction.java | 63 +++++++++++++------ 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java index 7bd33d79cb4..b48fc6e0777 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java @@ -46,32 +46,59 @@ public class RestNodesStatsAction extends BaseRestHandler { controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats", this); controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats", this); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/indices", new RestIndicesHandler()); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/indices", new RestIndicesHandler()); + RestIndicesHandler indicesHandler = new RestIndicesHandler(); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/indices", indicesHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/indices", indicesHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/indices/stats", indicesHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/indices/stats", indicesHandler); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/os", new RestOsHandler()); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/os", new RestOsHandler()); + RestOsHandler osHandler = new RestOsHandler(); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/os", osHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/os", osHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/os/stats", osHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/os/stats", osHandler); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/process", new RestProcessHandler()); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/process", new RestProcessHandler()); + RestProcessHandler processHandler = new RestProcessHandler(); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/process", processHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/process", processHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/process/stats", processHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/process/stats", processHandler); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/jvm", new RestJvmHandler()); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/jvm", new RestJvmHandler()); + RestJvmHandler jvmHandler = new RestJvmHandler(); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/jvm", jvmHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/jvm", jvmHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/jvm/stats", jvmHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/jvm/stats", jvmHandler); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/thread_pool", new RestThreadPoolHandler()); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/thread_pool", new RestThreadPoolHandler()); + RestThreadPoolHandler threadPoolHandler = new RestThreadPoolHandler(); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/thread_pool", threadPoolHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/thread_pool", threadPoolHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/thread_pool/stats", threadPoolHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/thread_pool/stats", threadPoolHandler); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/network", new RestNetworkHandler()); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/network", new RestNetworkHandler()); + RestNetworkHandler networkHandler = new RestNetworkHandler(); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/network", networkHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/network", networkHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/network/stats", networkHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/network/stats", networkHandler); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/fs", new RestFsHandler()); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/fs", new RestFsHandler()); + RestFsHandler fsHandler = new RestFsHandler(); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/fs", fsHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/fs", fsHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/fs/stats", fsHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/fs/stats", fsHandler); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/transport", new RestTransportHandler()); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/transport", new RestTransportHandler()); + RestTransportHandler transportHandler = new RestTransportHandler(); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/transport", transportHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/transport", transportHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/transport/stats", transportHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/transport/stats", transportHandler); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/http", new RestHttpHandler()); - controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/http", new RestHttpHandler()); + RestHttpHandler httpHandler = new RestHttpHandler(); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/stats/http", httpHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/stats/http", httpHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/http/stats", httpHandler); + controller.registerHandler(RestRequest.Method.GET, "/_nodes/{nodeId}/http/stats", httpHandler); } @Override From b160ddbb2ce7c2a6c7a3c3ec61a81d84530c2b34 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 7 Feb 2012 15:02:42 +0200 Subject: [PATCH 262/270] release 0.19.0.RC1 --- pom.xml | 2 +- src/main/java/org/elasticsearch/Version.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index d828aab5fed..38ee4c5b54a 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 org.elasticsearch elasticsearch - 0.19.0-SNAPSHOT + 0.19.0.RC1 jar ElasticSearch - Open Source, Distributed, RESTful Search Engine 2009 diff --git a/src/main/java/org/elasticsearch/Version.java b/src/main/java/org/elasticsearch/Version.java index 2a8d7d8bdab..c271725cdde 100644 --- a/src/main/java/org/elasticsearch/Version.java +++ b/src/main/java/org/elasticsearch/Version.java @@ -54,8 +54,8 @@ public class Version implements Serializable { public static final int V_0_18_8_ID = /*00*/180899; public static final Version V_0_18_8 = new Version(V_0_18_8_ID, false); - public static final int V_0_19_0_ID = /*00*/190099; - public static final Version V_0_19_0 = new Version(V_0_19_0_ID, true); + public static final int V_0_19_0_ID = /*00*/190051; + public static final Version V_0_19_0 = new Version(V_0_19_0_ID, false); public static final Version CURRENT = V_0_19_0; From a89878ce6c236c38d9c2271492e35943cc6367fe Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 7 Feb 2012 15:17:09 +0200 Subject: [PATCH 263/270] move to 0.19.0.RC2 snap --- pom.xml | 2 +- src/main/java/org/elasticsearch/Version.java | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 38ee4c5b54a..08cae670226 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 org.elasticsearch elasticsearch - 0.19.0.RC1 + 0.19.0.RC2-SNAPSHOT jar ElasticSearch - Open Source, Distributed, RESTful Search Engine 2009 diff --git a/src/main/java/org/elasticsearch/Version.java b/src/main/java/org/elasticsearch/Version.java index c271725cdde..023d628d6db 100644 --- a/src/main/java/org/elasticsearch/Version.java +++ b/src/main/java/org/elasticsearch/Version.java @@ -54,10 +54,13 @@ public class Version implements Serializable { public static final int V_0_18_8_ID = /*00*/180899; public static final Version V_0_18_8 = new Version(V_0_18_8_ID, false); - public static final int V_0_19_0_ID = /*00*/190051; - public static final Version V_0_19_0 = new Version(V_0_19_0_ID, false); + public static final int V_0_19_0_RC1_ID = /*00*/190051; + public static final Version V_0_19_0_RC1 = new Version(V_0_19_0_RC1_ID, false); - public static final Version CURRENT = V_0_19_0; + public static final int V_0_19_0_RC2_ID = /*00*/190052; + public static final Version V_0_19_0_RC2 = new Version(V_0_19_0_RC2_ID, true); + + public static final Version CURRENT = V_0_19_0_RC2; public static Version readVersion(StreamInput in) throws IOException { return fromId(in.readVInt()); @@ -84,8 +87,10 @@ public class Version implements Serializable { case V_0_18_8_ID: return V_0_18_8; - case V_0_19_0_ID: - return V_0_19_0; + case V_0_19_0_RC1_ID: + return V_0_19_0_RC1; + case V_0_19_0_RC2_ID: + return V_0_19_0_RC2; default: return new Version(id, null); } From 636c4ae8fa1690a0b449f5e66eb592802e301078 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 7 Feb 2012 19:19:33 +0200 Subject: [PATCH 264/270] Nodes Info API: Add `all` flag to return all data, closes #1676. --- .../admin/cluster/node/info/NodesInfoRequest.java | 15 +++++++++++++++ .../node/info/NodesInfoRequestBuilder.java | 8 ++++++++ .../cluster/node/info/RestNodesInfoAction.java | 4 ++++ 3 files changed, 27 insertions(+) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java index 5c882e8743e..cc5e0fb9aae 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java @@ -65,6 +65,21 @@ public class NodesInfoRequest extends NodesOperationRequest { return this; } + /** + * Sets to return all the data. + */ + public NodesInfoRequest all() { + settings = true; + os = true; + process = true; + jvm = true; + threadPool = true; + network = true; + transport = true; + http = true; + return this; + } + /** * Should the node settings be returned. */ diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java index 87723b48020..0bd380973fc 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java @@ -45,6 +45,14 @@ public class NodesInfoRequestBuilder extends BaseClusterRequestBuilder Date: Tue, 7 Feb 2012 20:15:55 +0200 Subject: [PATCH 265/270] upgrade to netty 3.3.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 08cae670226..62a6898bbf3 100644 --- a/pom.xml +++ b/pom.xml @@ -153,7 +153,7 @@ io.netty netty - 3.3.0.Final + 3.3.1.Final compile From 2ab3f533fe81058fe401cfd629ebf61ce0364c9c Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 7 Feb 2012 21:51:18 +0200 Subject: [PATCH 266/270] Index API: Don't wait for new mappings to be applied on the cluster by default, closes #1677. --- .../org/elasticsearch/action/index/TransportIndexAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java b/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java index c7d92a75b3e..fa1e4af6acb 100644 --- a/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java +++ b/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java @@ -85,7 +85,7 @@ public class TransportIndexAction extends TransportShardReplicationOperationActi this.mappingUpdatedAction = mappingUpdatedAction; this.autoCreateIndex = settings.getAsBoolean("action.auto_create_index", true); this.allowIdGeneration = settings.getAsBoolean("action.allow_id_generation", true); - this.waitForMappingChange = settings.getAsBoolean("action.wait_on_mapping_change", true); + this.waitForMappingChange = settings.getAsBoolean("action.wait_on_mapping_change", false); } @Override From c9107889a01ede7fa5d161798e10c4f2495b9719 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 7 Feb 2012 22:56:59 +0200 Subject: [PATCH 267/270] allow to enable last GC since in upcoming 1.7.0u4 the memory leak is fixed --- src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java b/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java index cd25a47f8b6..ba9bfb94827 100644 --- a/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java +++ b/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java @@ -91,7 +91,7 @@ public class JvmStats implements Streamable, Serializable, ToXContent { } } - JvmStats.enableLastGc = false; + JvmStats.enableLastGc = enableLastGc; } public static JvmStats jvmStats() { From 49fd3ff470be1ea51071099c32709519ada4efc3 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 7 Feb 2012 23:16:23 +0200 Subject: [PATCH 268/270] automatically enable last GC if higher or equal to 1.7.0 u4 --- .../elasticsearch/monitor/jvm/JvmInfo.java | 62 +++++++++++++++++++ .../elasticsearch/monitor/jvm/JvmStats.java | 10 ++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java b/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java index 9aea763ff8f..5b7cd07b717 100644 --- a/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java +++ b/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java @@ -123,6 +123,68 @@ public class JvmInfo implements Streamable, Serializable, ToXContent { return this.version; } + public int versionAsInteger() { + try { + int i = 0; + String sVersion = ""; + for (; i < version.length(); i++) { + if (!Character.isDigit(version.charAt(i)) && version.charAt(i) != '.') { + break; + } + if (version.charAt(i) != '.') { + sVersion += version.charAt(i); + } + } + if (i == 0) { + return -1; + } + return Integer.parseInt(sVersion); + } catch (Exception e) { + return -1; + } + } + + public int versionUpdatePack() { + try { + int i = 0; + String sVersion = ""; + for (; i < version.length(); i++) { + if (!Character.isDigit(version.charAt(i)) && version.charAt(i) != '.') { + break; + } + if (version.charAt(i) != '.') { + sVersion += version.charAt(i); + } + } + if (i == 0) { + return -1; + } + Integer.parseInt(sVersion); + int from; + if (version.charAt(i) == '_') { + // 1.7.0_4 + from = ++i; + } else if (version.charAt(i) == '-' && version.charAt(i + 1) == 'u') { + // 1.7.0-u2-b21 + i = i + 2; + from = i; + } else { + return -1; + } + for (; i < version.length(); i++) { + if (!Character.isDigit(version.charAt(i)) && version.charAt(i) != '.') { + break; + } + } + if (from == i) { + return -1; + } + return Integer.parseInt(version.substring(from, i)); + } catch (Exception e) { + return -1; + } + } + public String vmName() { return vmName; } diff --git a/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java b/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java index ba9bfb94827..c4c488b5e6d 100644 --- a/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java +++ b/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java @@ -66,7 +66,15 @@ public class JvmStats implements Streamable, Serializable, ToXContent { memoryMXBean = ManagementFactory.getMemoryMXBean(); threadMXBean = ManagementFactory.getThreadMXBean(); - boolean enableLastGc = Booleans.parseBoolean(System.getProperty("monitor.jvm.enable_last_gc"), false); + JvmInfo info = JvmInfo.jvmInfo(); + boolean defaultEnableLastGc = false; + if (info.versionAsInteger() == 170) { + defaultEnableLastGc = info.versionUpdatePack() >= 4; + } else if (info.versionAsInteger() > 170) { + defaultEnableLastGc = true; + } + + boolean enableLastGc = Booleans.parseBoolean(System.getProperty("monitor.jvm.enable_last_gc"), defaultEnableLastGc); if (enableLastGc) { try { Class sunGcClass = Class.forName("com.sun.management.GarbageCollectorMXBean"); From c75f8fc18aaab9f9c1b44768feb49909a2a5c301 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 8 Feb 2012 00:35:28 +0200 Subject: [PATCH 269/270] GC logging: Enable back automatic gc logging based on thresholds (even if last gc is not available), closes #1679. --- config/elasticsearch.yml | 10 ++ .../monitor/jvm/JvmMonitorService.java | 117 +++++++++++++++--- 2 files changed, 110 insertions(+), 17 deletions(-) diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index 4746cb5c085..cf967ea55c9 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -341,3 +341,13 @@ #index.search.slowlog.threshold.fetch.info: 800ms #index.search.slowlog.threshold.fetch.debug: 500ms #index.search.slowlog.threshold.fetch.trace: 200ms + +################################## GC Logging ################################ + +#monitor.jvm.gc.ParNew.warn: 1000ms +#monitor.jvm.gc.ParNew.info: 700ms +#monitor.jvm.gc.ParNew.debug: 400ms + +#monitor.jvm.gc.ConcurrentMarkSweep.warn: 10s +#monitor.jvm.gc.ConcurrentMarkSweep.info: 5s +#monitor.jvm.gc.ConcurrentMarkSweep.debug: 2s diff --git a/src/main/java/org/elasticsearch/monitor/jvm/JvmMonitorService.java b/src/main/java/org/elasticsearch/monitor/jvm/JvmMonitorService.java index 43313936907..dd31548e2ce 100644 --- a/src/main/java/org/elasticsearch/monitor/jvm/JvmMonitorService.java +++ b/src/main/java/org/elasticsearch/monitor/jvm/JvmMonitorService.java @@ -19,8 +19,10 @@ package org.elasticsearch.monitor.jvm; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -30,10 +32,10 @@ import org.elasticsearch.monitor.dump.DumpMonitorService; import org.elasticsearch.threadpool.ThreadPool; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.concurrent.ScheduledFuture; -import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; import static org.elasticsearch.monitor.dump.summary.SummaryDumpContributor.SUMMARY; import static org.elasticsearch.monitor.dump.thread.ThreadDumpContributor.THREAD_DUMP; @@ -54,21 +56,69 @@ public class JvmMonitorService extends AbstractLifecycleComponent gcThresholds; private volatile ScheduledFuture scheduledFuture; + static class GcThreshold { + public final String name; + public final long warnThreshold; + public final long infoThreshold; + public final long debugThreshold; + + GcThreshold(String name, long warnThreshold, long infoThreshold, long debugThreshold) { + this.name = name; + this.warnThreshold = warnThreshold; + this.infoThreshold = infoThreshold; + this.debugThreshold = debugThreshold; + } + + @Override + public String toString() { + return "GcThreshold{" + + "name='" + name + '\'' + + ", warnThreshold=" + warnThreshold + + ", infoThreshold=" + infoThreshold + + ", debugThreshold=" + debugThreshold + + '}'; + } + } + @Inject public JvmMonitorService(Settings settings, ThreadPool threadPool, DumpMonitorService dumpMonitorService) { super(settings); this.threadPool = threadPool; this.dumpMonitorService = dumpMonitorService; - this.enabled = componentSettings.getAsBoolean("enabled", JvmStats.isLastGcEnabled()); + this.enabled = componentSettings.getAsBoolean("enabled", true); this.interval = componentSettings.getAsTime("interval", timeValueSeconds(1)); - this.gcThreshold = componentSettings.getAsTime("gc_threshold", timeValueMillis(5000)); - logger.debug("enabled [{}], last_gc_enabled [{}], interval [{}], gc_threshold [{}]", enabled, JvmStats.isLastGcEnabled(), interval, gcThreshold); + MapBuilder gcThresholds = MapBuilder.newMapBuilder(); + Map gcThresholdGroups = componentSettings.getGroups("gc"); + for (Map.Entry entry : gcThresholdGroups.entrySet()) { + String name = entry.getKey(); + TimeValue warn = entry.getValue().getAsTime("warn", null); + TimeValue info = entry.getValue().getAsTime("info", null); + TimeValue debug = entry.getValue().getAsTime("debug", null); + if (warn == null || info == null || debug == null) { + logger.warn("ignoring gc_threshold for [{}], missing warn/info/debug values", name); + } else { + gcThresholds.put(name, new GcThreshold(name, warn.millis(), info.millis(), debug.millis())); + } + } + if (!gcThresholds.containsKey("ParNew")) { + gcThresholds.put("ParNew", new GcThreshold("ParNew", 1000, 700, 400)); + } + if (!gcThresholds.containsKey("ConcurrentMarkSweep")) { + gcThresholds.put("ConcurrentMarkSweep", new GcThreshold("ConcurrentMarkSweep", 10000, 5000, 2000)); + } + if (!gcThresholds.containsKey("default")) { + gcThresholds.put("default", new GcThreshold("default", 10000, 5000, 2000)); + } + + this.gcThresholds = gcThresholds.immutableMap(); + + logger.debug("enabled [{}], last_gc_enabled [{}], interval [{}], gc_threshold [{}]", enabled, JvmStats.isLastGcEnabled(), interval, this.gcThresholds); } @Override @@ -95,6 +145,8 @@ public class JvmMonitorService extends AbstractLifecycleComponent lastSeenDeadlocks = new HashSet(); public JvmMonitor() { @@ -106,14 +158,32 @@ public class JvmMonitorService extends AbstractLifecycleComponent 1) { continue; } - if (lastGc.duration().millis() > gcThreshold.millis()) { - logger.info("[gc][{}][{}] took [{}]/[{}], reclaimed [{}], leaving [{}] used, max [{}]", gc.name(), gc.getCollectionCount(), lastGc.duration(), gc.getCollectionTime(), lastGc.reclaimed(), lastGc.afterUsed(), lastGc.max()); - } else if (logger.isDebugEnabled()) { - logger.debug("[gc][{}][{}] took [{}]/[{}], reclaimed [{}], leaving [{}] used, max [{}]", gc.name(), gc.getCollectionCount(), lastGc.duration(), gc.getCollectionTime(), lastGc.reclaimed(), lastGc.afterUsed(), lastGc.max()); - } - } else { - long collectionTime = gc.collectionTime().millis() - lastJvmStats.gc().collectors()[i].collectionTime().millis(); - if (collectionTime > gcThreshold.millis()) { - logger.info("[gc][{}] collection occurred, took [{}]", gc.name(), new TimeValue(collectionTime)); + if (lastGc.duration().millis() > gcThreshold.warnThreshold) { + logger.warn("[last_gc][{}][{}][{}] duration [{}], collections [{}], total [{}]/[{}], reclaimed [{}], leaving [{}][{}]/[{}]", + gc.name(), seq, gc.getCollectionCount(), lastGc.duration(), collections, TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastGc.reclaimed(), lastGc.afterUsed(), lastGc.max()); + } else if (lastGc.duration().millis() > gcThreshold.infoThreshold) { + logger.info("[last_gc][{}][{}][{}] duration [{}], collections [{}], total [{}]/[{}], reclaimed [{}], leaving [{}]/[{}]", + gc.name(), seq, gc.getCollectionCount(), lastGc.duration(), collections, TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastGc.reclaimed(), lastGc.afterUsed(), lastGc.max()); + } else if (lastGc.duration().millis() > gcThreshold.debugThreshold && logger.isDebugEnabled()) { + logger.debug("[last_gc][{}][{}][{}] duration [{}], collections [{}], total [{}]/[{}], reclaimed [{}], leaving [{}]/[{}]", + gc.name(), seq, gc.getCollectionCount(), lastGc.duration(), collections, TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastGc.reclaimed(), lastGc.afterUsed(), lastGc.max()); } } + + long avgCollectionTime = collectionTime / collections; + + if (avgCollectionTime > gcThreshold.warnThreshold) { + logger.warn("[gc][{}][{}][{}] duration [{}], collections [{}], total [{}]/[{}], memory [{}]->[{}]/[{}]", + gc.name(), seq, gc.collectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastJvmStats.mem().heapUsed(), currentJvmStats.mem().heapUsed(), JvmInfo.jvmInfo().mem().heapMax()); + } else if (avgCollectionTime > gcThreshold.infoThreshold) { + logger.info("[gc][{}][{}][{}] duration [{}], collections [{}], total [{}]/[{}], memory [{}]->[{}]/[{}]", + gc.name(), seq, gc.collectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastJvmStats.mem().heapUsed(), currentJvmStats.mem().heapUsed(), JvmInfo.jvmInfo().mem().heapMax()); + } else if (avgCollectionTime > gcThreshold.debugThreshold && logger.isDebugEnabled()) { + logger.debug("[gc][{}][{}][{}] duration [{}], collections [{}], total [{}]/[{}], memory [{}]->[{}]/[{}]", + gc.name(), seq, gc.collectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastJvmStats.mem().heapUsed(), currentJvmStats.mem().heapUsed(), JvmInfo.jvmInfo().mem().heapMax()); + } } lastJvmStats = currentJvmStats; } From 23ca0d2cb25d3ba7a0ef2ae7d568bd3ecab2d026 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 8 Feb 2012 00:38:00 +0200 Subject: [PATCH 270/270] add also the interval that was checked for plain gc --- .../elasticsearch/monitor/jvm/JvmMonitorService.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/elasticsearch/monitor/jvm/JvmMonitorService.java b/src/main/java/org/elasticsearch/monitor/jvm/JvmMonitorService.java index dd31548e2ce..e658d7880b5 100644 --- a/src/main/java/org/elasticsearch/monitor/jvm/JvmMonitorService.java +++ b/src/main/java/org/elasticsearch/monitor/jvm/JvmMonitorService.java @@ -206,14 +206,14 @@ public class JvmMonitorService extends AbstractLifecycleComponent gcThreshold.warnThreshold) { - logger.warn("[gc][{}][{}][{}] duration [{}], collections [{}], total [{}]/[{}], memory [{}]->[{}]/[{}]", - gc.name(), seq, gc.collectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastJvmStats.mem().heapUsed(), currentJvmStats.mem().heapUsed(), JvmInfo.jvmInfo().mem().heapMax()); + logger.warn("[gc][{}][{}][{}] duration [{}], collections [{}]/[{}], total [{}]/[{}], memory [{}]->[{}]/[{}]", + gc.name(), seq, gc.collectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(currentJvmStats.timestamp() - lastJvmStats.timestamp()), TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastJvmStats.mem().heapUsed(), currentJvmStats.mem().heapUsed(), JvmInfo.jvmInfo().mem().heapMax()); } else if (avgCollectionTime > gcThreshold.infoThreshold) { - logger.info("[gc][{}][{}][{}] duration [{}], collections [{}], total [{}]/[{}], memory [{}]->[{}]/[{}]", - gc.name(), seq, gc.collectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastJvmStats.mem().heapUsed(), currentJvmStats.mem().heapUsed(), JvmInfo.jvmInfo().mem().heapMax()); + logger.info("[gc][{}][{}][{}] duration [{}], collections [{}]/[{}], total [{}]/[{}], memory [{}]->[{}]/[{}]", + gc.name(), seq, gc.collectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(currentJvmStats.timestamp() - lastJvmStats.timestamp()), TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastJvmStats.mem().heapUsed(), currentJvmStats.mem().heapUsed(), JvmInfo.jvmInfo().mem().heapMax()); } else if (avgCollectionTime > gcThreshold.debugThreshold && logger.isDebugEnabled()) { - logger.debug("[gc][{}][{}][{}] duration [{}], collections [{}], total [{}]/[{}], memory [{}]->[{}]/[{}]", - gc.name(), seq, gc.collectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastJvmStats.mem().heapUsed(), currentJvmStats.mem().heapUsed(), JvmInfo.jvmInfo().mem().heapMax()); + logger.debug("[gc][{}][{}][{}] duration [{}], collections [{}]/[{}], total [{}]/[{}], memory [{}]->[{}]/[{}]", + gc.name(), seq, gc.collectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(currentJvmStats.timestamp() - lastJvmStats.timestamp()), TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastJvmStats.mem().heapUsed(), currentJvmStats.mem().heapUsed(), JvmInfo.jvmInfo().mem().heapMax()); } } lastJvmStats = currentJvmStats;