From 4fad18d3bc6b0227d51e0d8b88a32c0373f525b5 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 19 Mar 2016 11:26:40 -0400 Subject: [PATCH 01/45] Fix line-length violations in InstallPluginCommand This commit fixes the line-length checkstyle violations in InstallPluginCommand.java and removes this from the list of files for which the line-length check is suppressed. --- .../resources/checkstyle_suppressions.xml | 1 - .../plugins/InstallPluginCommand.java | 24 ++++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 87c049ae0b1..eee82afe116 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -668,7 +668,6 @@ - diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index 18e996f6f37..c8fa426b6a4 100644 --- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -182,11 +182,18 @@ class InstallPluginCommand extends Command { final String version = Version.CURRENT.toString(); final String url; if (System.getProperty(PROPERTY_SUPPORT_STAGING_URLS, "false").equals("true")) { - url = String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/staging/%1$s-%2$s/org/elasticsearch/plugin/%3$s/%1$s/%3$s-%1$s.zip", - version, Build.CURRENT.shortHash(), pluginId); + url = String.format( + Locale.ROOT, + "https://download.elastic.co/elasticsearch/staging/%1$s-%2$s/org/elasticsearch/plugin/%3$s/%1$s/%3$s-%1$s.zip", + version, + Build.CURRENT.shortHash(), + pluginId); } else { - url = String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/%1$s/%2$s/%1$s-%2$s.zip", - pluginId, version); + url = String.format( + Locale.ROOT, + "https://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/%1$s/%2$s/%1$s-%2$s.zip", + pluginId, + version); } terminal.println("-> Downloading " + pluginId + " from elastic"); return downloadZipAndChecksum(url, tmpDir); @@ -366,7 +373,10 @@ class InstallPluginCommand extends Command { final Path destination = env.pluginsFile().resolve(info.getName()); if (Files.exists(destination)) { - throw new UserError(ExitCodes.USAGE, "plugin directory " + destination.toAbsolutePath() + " already exists. To update the plugin, uninstall it first using 'remove " + info.getName() + "' command"); + throw new UserError( + ExitCodes.USAGE, + "plugin directory " + destination.toAbsolutePath() + + " already exists. To update the plugin, uninstall it first using 'remove " + info.getName() + "' command"); } Path tmpBinDir = tmpRoot.resolve("bin"); @@ -419,7 +429,9 @@ class InstallPluginCommand extends Command { try (DirectoryStream stream = Files.newDirectoryStream(tmpBinDir)) { for (Path srcFile : stream) { if (Files.isDirectory(srcFile)) { - throw new UserError(ExitCodes.DATA_ERROR, "Directories not allowed in bin dir for plugin " + info.getName() + ", found " + srcFile.getFileName()); + throw new UserError( + ExitCodes.DATA_ERROR, + "Directories not allowed in bin dir for plugin " + info.getName() + ", found " + srcFile.getFileName()); } Path destFile = destBinDir.resolve(tmpBinDir.relativize(srcFile)); From 554eb8aa87fbd7d516b8b30b88d5008b04b91796 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 19 Mar 2016 11:32:06 -0400 Subject: [PATCH 02/45] Refactor POSIX handling when installing plugins This commit refactors the handling of POSIX filesystem attributes when install plugins. --- .../plugins/InstallPluginCommand.java | 79 ++++++++++--------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index c8fa426b6a4..c1212e46238 100644 --- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -21,7 +21,6 @@ package org.elasticsearch.plugins; import joptsimple.OptionSet; import joptsimple.OptionSpec; -import org.apache.lucene.util.Constants; import org.apache.lucene.util.IOUtils; import org.elasticsearch.Build; import org.elasticsearch.Version; @@ -56,6 +55,8 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; +import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -249,21 +250,8 @@ class InstallPluginCommand extends Command { private Path unzip(Path zip, Path pluginsDir) throws IOException, UserError { // unzip plugin to a staging temp dir - final Path target; - if (Constants.WINDOWS) { - target = Files.createTempDirectory(pluginsDir, ".installing-"); - } else { - Set perms = new HashSet<>(); - perms.add(PosixFilePermission.OWNER_EXECUTE); - perms.add(PosixFilePermission.OWNER_READ); - perms.add(PosixFilePermission.OWNER_WRITE); - perms.add(PosixFilePermission.GROUP_READ); - perms.add(PosixFilePermission.GROUP_EXECUTE); - perms.add(PosixFilePermission.OTHERS_READ); - perms.add(PosixFilePermission.OTHERS_EXECUTE); - target = Files.createTempDirectory(pluginsDir, ".installing-", PosixFilePermissions.asFileAttribute(perms)); - } - Files.createDirectories(target); + + final Path target = stagingDirectory(pluginsDir); boolean hasEsDir = false; // TODO: we should wrap this in a try/catch and try deleting the target dir on failure? @@ -308,6 +296,22 @@ class InstallPluginCommand extends Command { return target; } + private Path stagingDirectory(Path pluginsDir) throws IOException { + try { + Set perms = new HashSet<>(); + perms.add(PosixFilePermission.OWNER_EXECUTE); + perms.add(PosixFilePermission.OWNER_READ); + perms.add(PosixFilePermission.OWNER_WRITE); + perms.add(PosixFilePermission.GROUP_READ); + perms.add(PosixFilePermission.GROUP_EXECUTE); + perms.add(PosixFilePermission.OTHERS_READ); + perms.add(PosixFilePermission.OTHERS_EXECUTE); + return Files.createTempDirectory(pluginsDir, ".installing-", PosixFilePermissions.asFileAttribute(perms)); + } catch (UnsupportedOperationException e) { + return Files.createTempDirectory(pluginsDir, ".installing-"); + } + } + /** Load information about the plugin, and verify it can be installed with no errors. */ private PluginInfo verify(Terminal terminal, Path pluginRoot, boolean isBatch) throws Exception { // read and validate the plugin descriptor @@ -413,17 +417,15 @@ class InstallPluginCommand extends Command { } Files.createDirectory(destBinDir); - Set perms = new HashSet<>(); - if (Constants.WINDOWS == false) { - // setup file attributes for the installed files to those of the parent dir - PosixFileAttributeView binAttrs = Files.getFileAttributeView(destBinDir.getParent(), PosixFileAttributeView.class); - if (binAttrs != null) { - perms = new HashSet<>(binAttrs.readAttributes().permissions()); - // setting execute bits, since this just means "the file is executable", and actual execution requires read - perms.add(PosixFilePermission.OWNER_EXECUTE); - perms.add(PosixFilePermission.GROUP_EXECUTE); - perms.add(PosixFilePermission.OTHERS_EXECUTE); - } + // setup file attributes for the installed files to those of the parent dir + final Set perms = new HashSet<>(); + final PosixFileAttributeView binAttributeView = Files.getFileAttributeView(destBinDir.getParent(), PosixFileAttributeView.class); + if (binAttributeView != null) { + perms.addAll(binAttributeView.readAttributes().permissions()); + // setting execute bits, since this just means "the file is executable", and actual execution requires read + perms.add(PosixFilePermission.OWNER_EXECUTE); + perms.add(PosixFilePermission.GROUP_EXECUTE); + perms.add(PosixFilePermission.OTHERS_EXECUTE); } try (DirectoryStream stream = Files.newDirectoryStream(tmpBinDir)) { @@ -437,8 +439,8 @@ class InstallPluginCommand extends Command { Path destFile = destBinDir.resolve(tmpBinDir.relativize(srcFile)); Files.copy(srcFile, destFile); - if (perms.isEmpty() == false) { - PosixFileAttributeView view = Files.getFileAttributeView(destFile, PosixFileAttributeView.class); + final PosixFileAttributeView view = Files.getFileAttributeView(destFile, PosixFileAttributeView.class); + if (view != null) { view.setPermissions(perms); } } @@ -457,15 +459,12 @@ class InstallPluginCommand extends Command { // create the plugin's config dir "if necessary" Files.createDirectories(destConfigDir); - - final PosixFileAttributes destConfigDirAttributes; - if (Constants.WINDOWS) { - destConfigDirAttributes = null; - } else { - destConfigDirAttributes = - Files.getFileAttributeView(destConfigDir.getParent(), PosixFileAttributeView.class).readAttributes(); + final PosixFileAttributeView destConfigDirAttributesView = + Files.getFileAttributeView(destConfigDir.getParent(), PosixFileAttributeView.class); + final PosixFileAttributes destConfigDirAttributes = + destConfigDirAttributesView != null ? destConfigDirAttributesView.readAttributes() : null; + if (destConfigDirAttributes != null) { setOwnerGroup(destConfigDir, destConfigDirAttributes); - } try (DirectoryStream stream = Files.newDirectoryStream(tmpConfigDir)) { @@ -477,7 +476,7 @@ class InstallPluginCommand extends Command { Path destFile = destConfigDir.resolve(tmpConfigDir.relativize(srcFile)); if (Files.exists(destFile) == false) { Files.copy(srcFile, destFile); - if (Constants.WINDOWS == false) { + if (destConfigDirAttributes != null) { setOwnerGroup(destFile, destConfigDirAttributes); } } @@ -486,8 +485,10 @@ class InstallPluginCommand extends Command { IOUtils.rm(tmpConfigDir); // clean up what we just copied } - private static void setOwnerGroup(Path path, PosixFileAttributes attributes) throws IOException { + private static void setOwnerGroup(final Path path, final PosixFileAttributes attributes) throws IOException { + Objects.requireNonNull(attributes); PosixFileAttributeView fileAttributeView = Files.getFileAttributeView(path, PosixFileAttributeView.class); + assert fileAttributeView != null; fileAttributeView.setOwner(attributes.owner()); fileAttributeView.setGroup(attributes.group()); } From 6db6c15d0662f6fc310a3f97eddddd3b9c432aec Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sun, 20 Mar 2016 17:47:39 -0400 Subject: [PATCH 03/45] Add tests of POSIX handling for installing plugins This commit refactors the unit tests for installing plugins to test against mock filesystems (as well as the native filesystem) for better test coverage. This commit also adds tests that cover the POSIX attributes handling when installing plugins (e.g., ensuring that the plugins directory has the right permissions, the bin directory has execute permissions, and the config directory has the same owner and group as its parent). --- .../elasticsearch/common/io/PathUtils.java | 36 ++- .../common/settings/Setting.java | 1 + .../org/elasticsearch/env/Environment.java | 15 +- .../plugins/InstallPluginCommand.java | 4 +- .../InternalSettingsPreparerTests.java | 1 + .../plugins/InstallPluginCommandTests.java | 243 ++++++++++++------ 6 files changed, 213 insertions(+), 87 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/io/PathUtils.java b/core/src/main/java/org/elasticsearch/common/io/PathUtils.java index 0002c1e5b5f..ab5b46f3596 100644 --- a/core/src/main/java/org/elasticsearch/common/io/PathUtils.java +++ b/core/src/main/java/org/elasticsearch/common/io/PathUtils.java @@ -27,7 +27,7 @@ import java.nio.file.FileSystems; import java.nio.file.Path; import java.nio.file.Paths; -/** +/** * Utilities for creating a Path from names, * or accessing the default FileSystem. *

@@ -39,14 +39,14 @@ import java.nio.file.Paths; public final class PathUtils { /** no instantiation */ private PathUtils() {} - + /** the actual JDK default */ static final FileSystem ACTUAL_DEFAULT = FileSystems.getDefault(); - + /** can be changed by tests */ static volatile FileSystem DEFAULT = ACTUAL_DEFAULT; - - /** + + /** * Returns a {@code Path} from name components. *

* This works just like {@code Paths.get()}. @@ -57,10 +57,30 @@ public final class PathUtils { * a path against an existing one! */ public static Path get(String first, String... more) { - return DEFAULT.getPath(first, more); + return get(DEFAULT, first, more); } - - /** + + /** + * Returns a {@code Path} from name components against the given + * {@code FileSystem}. + *

+ * This works just like {@code Paths.get()}. + * Remember: just like {@code Paths.get()} this is NOT A STRING CONCATENATION + * UTILITY FUNCTION. + *

+ * Remember: this should almost never be used. Usually resolve + * a path against an existing one! + * + * @param fs the given {@code FileSystem} + * @param first the first path component + * @param more the remaining path components + * @return a path + */ + public static Path get(FileSystem fs, String first, String... more) { + return fs.getPath(first, more); + } + + /** * Returns a {@code Path} from a URI *

* This works just like {@code Paths.get()}. diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java index f0e1b2e64ea..e600f845d86 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; +import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; diff --git a/core/src/main/java/org/elasticsearch/env/Environment.java b/core/src/main/java/org/elasticsearch/env/Environment.java index e022ce6ad2f..ca7363a32ee 100644 --- a/core/src/main/java/org/elasticsearch/env/Environment.java +++ b/core/src/main/java/org/elasticsearch/env/Environment.java @@ -31,6 +31,7 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.nio.file.FileStore; +import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -108,28 +109,32 @@ public class Environment { } public Environment(Settings settings) { + this(PathUtils.getDefaultFileSystem(), settings); + } + + public Environment(FileSystem fs, Settings settings) { this.settings = settings; final Path homeFile; if (PATH_HOME_SETTING.exists(settings)) { - homeFile = PathUtils.get(cleanPath(PATH_HOME_SETTING.get(settings))); + homeFile = PathUtils.get(fs, PATH_HOME_SETTING.get(settings)); } else { throw new IllegalStateException(PATH_HOME_SETTING.getKey() + " is not configured"); } if (PATH_CONF_SETTING.exists(settings)) { - configFile = PathUtils.get(cleanPath(PATH_CONF_SETTING.get(settings))); + configFile = PathUtils.get(fs, cleanPath(PATH_CONF_SETTING.get(settings))); } else { configFile = homeFile.resolve("config"); } if (PATH_SCRIPTS_SETTING.exists(settings)) { - scriptsFile = PathUtils.get(cleanPath(PATH_SCRIPTS_SETTING.get(settings))); + scriptsFile = PathUtils.get(fs, cleanPath(PATH_SCRIPTS_SETTING.get(settings))); } else { scriptsFile = configFile.resolve("scripts"); } if (PATH_PLUGINS_SETTING.exists(settings)) { - pluginsFile = PathUtils.get(cleanPath(PATH_PLUGINS_SETTING.get(settings))); + pluginsFile = PathUtils.get(fs, cleanPath(PATH_PLUGINS_SETTING.get(settings))); } else { pluginsFile = homeFile.resolve("plugins"); } @@ -139,7 +144,7 @@ public class Environment { dataFiles = new Path[dataPaths.size()]; dataWithClusterFiles = new Path[dataPaths.size()]; for (int i = 0; i < dataPaths.size(); i++) { - dataFiles[i] = PathUtils.get(dataPaths.get(i)); + dataFiles[i] = PathUtils.get(fs, dataPaths.get(i)); dataWithClusterFiles[i] = dataFiles[i].resolve(ClusterName.clusterNameFromSettings(settings).value()); } } else { diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index c1212e46238..e822021e0b6 100644 --- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -307,7 +307,7 @@ class InstallPluginCommand extends Command { perms.add(PosixFilePermission.OTHERS_READ); perms.add(PosixFilePermission.OTHERS_EXECUTE); return Files.createTempDirectory(pluginsDir, ".installing-", PosixFilePermissions.asFileAttribute(perms)); - } catch (UnsupportedOperationException e) { + } catch (IllegalArgumentException | UnsupportedOperationException e) { return Files.createTempDirectory(pluginsDir, ".installing-"); } } @@ -338,7 +338,7 @@ class InstallPluginCommand extends Command { } /** check a candidate plugin for jar hell before installing it */ - private void jarHellCheck(Path candidate, Path pluginsDir, boolean isolated) throws Exception { + void jarHellCheck(Path candidate, Path pluginsDir, boolean isolated) throws Exception { // create list of current jars in classpath final List jars = new ArrayList<>(); jars.addAll(Arrays.asList(JarHell.parseClassPath())); diff --git a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java index c979b2f4013..8f12ebc4597 100644 --- a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java +++ b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.node.internal; import java.io.IOException; import java.io.InputStream; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index fb69c817f3a..7c516863637 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -19,6 +19,19 @@ package org.elasticsearch.plugins; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.Version; +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.cli.UserError; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.PosixPermissionsResetter; + import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; @@ -26,6 +39,7 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystem; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.NoSuchFileException; @@ -36,44 +50,85 @@ import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.PosixFileAttributeView; import java.nio.file.attribute.PosixFileAttributes; import java.nio.file.attribute.PosixFilePermission; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.Version; -import org.elasticsearch.cli.MockTerminal; -import org.elasticsearch.cli.UserError; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.PosixPermissionsResetter; -import org.junit.BeforeClass; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.containsInAnyOrder; @LuceneTestCase.SuppressFileSystems("*") public class InstallPluginCommandTests extends ESTestCase { - private static boolean isPosix; + private final Function temp; - @BeforeClass - public static void checkPosix() throws IOException { - isPosix = Files.getFileAttributeView(createTempFile(), PosixFileAttributeView.class) != null; + private final FileSystem fs; + private final boolean isPosix; + private final boolean isReal; + + public InstallPluginCommandTests(FileSystem fs, Function temp) { + this.fs = fs; + this.temp = temp; + this.isPosix = fs.supportedFileAttributeViews().contains("posix"); + this.isReal = fs == PathUtils.getDefaultFileSystem(); + } + + @ParametersFactory + public static Iterable parameters() { + class Parameter { + private final FileSystem fileSystem; + private final Function temp; + + public Parameter(FileSystem fileSystem, Supplier root) { + this(fileSystem, root, s -> { + try { + return Files.createTempDirectory(fileSystem.getPath(root.get()), s); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + public Parameter(FileSystem fileSystem, Supplier root, Function temp) { + this.fileSystem = fileSystem; + this.temp = temp; + } + } + List parameters = new ArrayList<>(); + parameters.add(new Parameter(Jimfs.newFileSystem(Configuration.windows()), () -> "c:\\")); + parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.osX())), () -> "/")); + parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.unix())), () -> "/")); + parameters.add(new Parameter(PathUtils.getDefaultFileSystem(), () -> createTempDir().toString(), LuceneTestCase::createTempDir )); + return parameters.stream().map(p -> new Object[] { p.fileSystem, p.temp }).collect(Collectors.toList()); + } + + private static Configuration toPosix(Configuration configuration) { + return configuration.toBuilder().setAttributeViews("basic", "owner", "posix", "unix").build(); } /** Creates a test environment with bin, config and plugins directories. */ - static Environment createEnv() throws IOException { - Path home = createTempDir(); + static Environment createEnv(FileSystem fs, Function temp) throws IOException { + Path home = temp.apply("install-plugin-command-tests"); Files.createDirectories(home.resolve("bin")); Files.createFile(home.resolve("bin").resolve("elasticsearch")); Files.createDirectories(home.resolve("config")); Files.createFile(home.resolve("config").resolve("elasticsearch.yml")); - Files.createDirectories(home.resolve("plugins")); + Path plugins = Files.createDirectories(home.resolve("plugins")); + assertTrue(Files.exists(plugins)); Settings settings = Settings.builder() .put("path.home", home) .build(); - return new Environment(settings); + return new Environment(fs, settings); + } + + static Path createPluginDir(Function temp) throws IOException { + return temp.apply("pluginDir"); } /** creates a fake jar file with empty class files */ @@ -115,14 +170,40 @@ public class InstallPluginCommandTests extends ESTestCase { } static MockTerminal installPlugin(String pluginUrl, Environment env) throws Exception { + return installPlugin(pluginUrl, env, false); + } + + static MockTerminal installPlugin(String pluginUrl, Environment env, boolean jarHellCheck) throws Exception { MockTerminal terminal = new MockTerminal(); - new InstallPluginCommand(env).execute(terminal, pluginUrl, true); + new InstallPluginCommand(env) { + @Override + void jarHellCheck(Path candidate, Path pluginsDir, boolean isolated) throws Exception { + if (jarHellCheck) { + super.jarHellCheck(candidate, pluginsDir, isolated); + } + } + }.execute(terminal, pluginUrl, true); return terminal; } void assertPlugin(String name, Path original, Environment env) throws IOException { Path got = env.pluginsFile().resolve(name); assertTrue("dir " + name + " exists", Files.exists(got)); + + if (isPosix) { + Set perms = Files.getPosixFilePermissions(got); + assertThat( + perms, + containsInAnyOrder( + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, + PosixFilePermission.OWNER_EXECUTE, + PosixFilePermission.GROUP_READ, + PosixFilePermission.GROUP_EXECUTE, + PosixFilePermission.OTHERS_READ, + PosixFilePermission.OTHERS_EXECUTE)); + } + assertTrue("jar was copied", Files.exists(got.resolve("plugin.jar"))); assertFalse("bin was not copied", Files.exists(got.resolve("bin"))); assertFalse("config was not copied", Files.exists(got.resolve("config"))); @@ -152,6 +233,16 @@ public class InstallPluginCommandTests extends ESTestCase { Path configDir = env.configFile().resolve(name); assertTrue("config dir exists", Files.exists(configDir)); assertTrue("config is a dir", Files.isDirectory(configDir)); + + if (isPosix) { + Path configRoot = env.configFile(); + PosixFileAttributes configAttributes = + Files.getFileAttributeView(configRoot, PosixFileAttributeView.class).readAttributes(); + PosixFileAttributes attributes = Files.getFileAttributeView(configDir, PosixFileAttributeView.class).readAttributes(); + assertThat(attributes.owner(), equalTo(configAttributes.owner())); + assertThat(attributes.group(), equalTo(configAttributes.group())); + } + try (DirectoryStream stream = Files.newDirectoryStream(configDir)) { for (Path file : stream) { assertFalse("not a dir", Files.isDirectory(file)); @@ -172,16 +263,16 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testSomethingWorks() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); String pluginZip = createPlugin("fake", pluginDir); installPlugin(pluginZip, env); assertPlugin("fake", pluginDir, env); } public void testSpaceInUrl() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); String pluginZip = createPlugin("fake", pluginDir); Path pluginZipWithSpaces = createTempFile("foo bar", ".zip"); try (InputStream in = new URL(pluginZip).openStream()) { @@ -192,28 +283,30 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testMalformedUrlNotMaven() throws Exception { + Environment env = createEnv(fs, temp); // has two colons, so it appears similar to maven coordinates MalformedURLException e = expectThrows(MalformedURLException.class, () -> { - installPlugin("://host:1234", createEnv()); + installPlugin("://host:1234", env); }); assertTrue(e.getMessage(), e.getMessage().contains("no protocol")); } public void testPluginsDirMissing() throws Exception { - Environment env = createEnv(); + Environment env = createEnv(fs, temp); Files.delete(env.pluginsFile()); - Path pluginDir = createTempDir(); + Path pluginDir = createPluginDir(temp); String pluginZip = createPlugin("fake", pluginDir); installPlugin(pluginZip, env); assertPlugin("fake", pluginDir, env); } public void testPluginsDirReadOnly() throws Exception { - assumeTrue("posix filesystem", isPosix); - Environment env = createEnv(); + assumeTrue("posix and filesystem", isPosix && isReal); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); try (PosixPermissionsResetter pluginsAttrs = new PosixPermissionsResetter(env.pluginsFile())) { pluginsAttrs.setPermissions(new HashSet<>()); - String pluginZip = createPlugin("fake", createTempDir()); + String pluginZip = createPlugin("fake", pluginDir); IOException e = expectThrows(IOException.class, () -> { installPlugin(pluginZip, env); }); @@ -223,8 +316,9 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testBuiltinModule() throws Exception { - Environment env = createEnv(); - String pluginZip = createPlugin("lang-groovy", createTempDir()); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); + String pluginZip = createPlugin("lang-groovy", pluginDir); UserError e = expectThrows(UserError.class, () -> { installPlugin(pluginZip, env); }); @@ -233,24 +327,26 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testJarHell() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); - writeJar(pluginDir.resolve("other.jar"), "FakePlugin"); - String pluginZip = createPlugin("fake", pluginDir); // adds plugin.jar with FakePlugin + // jar hell test needs a real filesystem + assumeTrue("real filesystem", isReal); + Environment environment = createEnv(fs, temp); + Path pluginDirectory = createPluginDir(temp); + writeJar(pluginDirectory.resolve("other.jar"), "FakePlugin"); + String pluginZip = createPlugin("fake", pluginDirectory); // adds plugin.jar with FakePlugin IllegalStateException e = expectThrows(IllegalStateException.class, () -> { - installPlugin(pluginZip, env); + installPlugin(pluginZip, environment, true); }); assertTrue(e.getMessage(), e.getMessage().contains("jar hell")); - assertInstallCleaned(env); + assertInstallCleaned(environment); } public void testIsolatedPlugins() throws Exception { - Environment env = createEnv(); + Environment env = createEnv(fs, temp); // these both share the same FakePlugin class - Path pluginDir1 = createTempDir(); + Path pluginDir1 = createPluginDir(temp); String pluginZip1 = createPlugin("fake1", pluginDir1); installPlugin(pluginZip1, env); - Path pluginDir2 = createTempDir(); + Path pluginDir2 = createPluginDir(temp); String pluginZip2 = createPlugin("fake2", pluginDir2); installPlugin(pluginZip2, env); assertPlugin("fake1", pluginDir1, env); @@ -258,8 +354,9 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testPurgatoryJarHell() throws Exception { - Environment env = createEnv(); - Path pluginDir1 = createTempDir(); + assumeTrue("real filesystem", isReal); + Environment environment = createEnv(fs, temp); + Path pluginDir1 = createPluginDir(temp); PluginTestUtil.writeProperties(pluginDir1, "description", "fake desc", "name", "fake1", @@ -270,9 +367,9 @@ public class InstallPluginCommandTests extends ESTestCase { "isolated", "false"); writeJar(pluginDir1.resolve("plugin.jar"), "FakePlugin"); String pluginZip1 = writeZip(pluginDir1, "elasticsearch"); - installPlugin(pluginZip1, env); + installPlugin(pluginZip1, environment); - Path pluginDir2 = createTempDir(); + Path pluginDir2 = createPluginDir(temp); PluginTestUtil.writeProperties(pluginDir2, "description", "fake desc", "name", "fake2", @@ -284,15 +381,16 @@ public class InstallPluginCommandTests extends ESTestCase { writeJar(pluginDir2.resolve("plugin.jar"), "FakePlugin"); String pluginZip2 = writeZip(pluginDir2, "elasticsearch"); IllegalStateException e = expectThrows(IllegalStateException.class, () -> { - installPlugin(pluginZip2, env); + installPlugin(pluginZip2, environment, true); }); assertTrue(e.getMessage(), e.getMessage().contains("jar hell")); - assertInstallCleaned(env); + assertInstallCleaned(environment); } public void testExistingPlugin() throws Exception { - Environment env = createEnv(); - String pluginZip = createPlugin("fake", createTempDir()); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); + String pluginZip = createPlugin("fake", pluginDir); installPlugin(pluginZip, env); UserError e = expectThrows(UserError.class, () -> { installPlugin(pluginZip, env); @@ -302,8 +400,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testBin() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path binDir = pluginDir.resolve("bin"); Files.createDirectory(binDir); Files.createFile(binDir.resolve("somescript")); @@ -313,8 +411,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testBinNotDir() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path binDir = pluginDir.resolve("bin"); Files.createFile(binDir); String pluginZip = createPlugin("fake", pluginDir); @@ -326,8 +424,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testBinContainsDir() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path dirInBinDir = pluginDir.resolve("bin").resolve("foo"); Files.createDirectories(dirInBinDir); Files.createFile(dirInBinDir.resolve("somescript")); @@ -340,8 +438,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testBinConflict() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path binDir = pluginDir.resolve("bin"); Files.createDirectory(binDir); Files.createFile(binDir.resolve("somescript")); @@ -355,8 +453,8 @@ public class InstallPluginCommandTests extends ESTestCase { public void testBinPermissions() throws Exception { assumeTrue("posix filesystem", isPosix); - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path binDir = pluginDir.resolve("bin"); Files.createDirectory(binDir); Files.createFile(binDir.resolve("somescript")); @@ -372,8 +470,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testConfig() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path configDir = pluginDir.resolve("config"); Files.createDirectory(configDir); Files.createFile(configDir.resolve("custom.yaml")); @@ -383,11 +481,11 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testExistingConfig() throws Exception { - Environment env = createEnv(); + Environment env = createEnv(fs, temp); Path envConfigDir = env.configFile().resolve("fake"); Files.createDirectories(envConfigDir); Files.write(envConfigDir.resolve("custom.yaml"), "existing config".getBytes(StandardCharsets.UTF_8)); - Path pluginDir = createTempDir(); + Path pluginDir = createPluginDir(temp); Path configDir = pluginDir.resolve("config"); Files.createDirectory(configDir); Files.write(configDir.resolve("custom.yaml"), "new config".getBytes(StandardCharsets.UTF_8)); @@ -402,8 +500,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testConfigNotDir() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path configDir = pluginDir.resolve("config"); Files.createFile(configDir); String pluginZip = createPlugin("fake", pluginDir); @@ -415,8 +513,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testConfigContainsDir() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path dirInConfigDir = pluginDir.resolve("config").resolve("foo"); Files.createDirectories(dirInConfigDir); Files.createFile(dirInConfigDir.resolve("myconfig.yml")); @@ -429,8 +527,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testConfigConflict() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path configDir = pluginDir.resolve("config"); Files.createDirectory(configDir); Files.createFile(configDir.resolve("myconfig.yml")); @@ -443,8 +541,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testMissingDescriptor() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Files.createFile(pluginDir.resolve("fake.yml")); String pluginZip = writeZip(pluginDir, "elasticsearch"); NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> { @@ -455,8 +553,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testMissingDirectory() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Files.createFile(pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES)); String pluginZip = writeZip(pluginDir, null); UserError e = expectThrows(UserError.class, () -> { @@ -467,13 +565,14 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testZipRelativeOutsideEntryName() throws Exception { + Environment env = createEnv(fs, temp); Path zip = createTempDir().resolve("broken.zip"); try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(zip))) { stream.putNextEntry(new ZipEntry("elasticsearch/../blah")); } String pluginZip = zip.toUri().toURL().toString(); IOException e = expectThrows(IOException.class, () -> { - installPlugin(pluginZip, createEnv()); + installPlugin(pluginZip, env); }); assertTrue(e.getMessage(), e.getMessage().contains("resolving outside of plugin directory")); } From 98123e97754a6db24c57606a0d14f6401b7b792b Mon Sep 17 00:00:00 2001 From: jaymode Date: Mon, 21 Mar 2016 09:30:20 -0400 Subject: [PATCH 04/45] Implement available for all StreamInput classes There are some implementation of StreamInput that implement the available method and there are others that do not implement this method. This change makes the available method abstract in the StreamInput class and implements the method where it was not previously implemented. --- .../common/bytes/PagedBytesReference.java | 5 ++++ .../common/io/stream/FilterStreamInput.java | 5 ++++ .../io/stream/InputStreamStreamInput.java | 5 ++++ .../common/io/stream/StreamInput.java | 6 ++-- .../bytes/PagedBytesReferenceTests.java | 11 +++++++- .../common/io/stream/BytesStreamsTests.java | 7 ++++- .../common/io/stream/StreamTests.java | 28 +++++++++++++++++++ 7 files changed, 62 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java index a41b0388af7..16ce91dc38f 100644 --- a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java +++ b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java @@ -445,5 +445,10 @@ public class PagedBytesReference implements BytesReference { // do nothing } + @Override + public int available() throws IOException { + return length - pos; + } + } } diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java index 5f3bd011dd9..b8132b4e870 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java @@ -59,6 +59,11 @@ public abstract class FilterStreamInput extends StreamInput { delegate.close(); } + @Override + public int available() throws IOException { + return delegate.available(); + } + @Override public Version getVersion() { return delegate.getVersion(); diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java index e9aa52cf4d0..d786041af49 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java @@ -74,6 +74,11 @@ public class InputStreamStreamInput extends StreamInput { is.close(); } + @Override + public int available() throws IOException { + return is.available(); + } + @Override public int read() throws IOException { return is.read(); diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index aca136a2a9a..9bb10fa677f 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -37,10 +37,8 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.text.Text; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; -import org.elasticsearch.ingest.IngestStats; import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.suggest.SuggestionBuilder; -import org.elasticsearch.search.suggest.completion.context.QueryContext; import org.elasticsearch.search.suggest.phrase.SmoothingModel; import org.elasticsearch.tasks.Task; import org.elasticsearch.search.aggregations.AggregatorBuilder; @@ -68,7 +66,6 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; import java.util.function.Supplier; import static org.elasticsearch.ElasticsearchException.readException; @@ -375,6 +372,9 @@ public abstract class StreamInput extends InputStream { @Override public abstract void close() throws IOException; + @Override + public abstract int available() throws IOException; + public String[] readStringArray() throws IOException { int size = readVInt(); if (size == 0) { diff --git a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java index 95a65f82924..7b34685da7d 100644 --- a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java +++ b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java @@ -154,7 +154,7 @@ public class PagedBytesReferenceTests extends ESTestCase { } public void testStreamInputBulkReadWithOffset() throws IOException { - int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20)); + final int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20)); BytesReference pbr = getRandomizedPagedBytesReference(length); StreamInput si = pbr.streamInput(); assertNotNull(si); @@ -162,6 +162,7 @@ public class PagedBytesReferenceTests extends ESTestCase { // read a bunch of single bytes one by one int offset = randomIntBetween(1, length / 2); for (int i = 0; i < offset; i++) { + assertEquals(si.available(), length - i); assertEquals(pbr.get(i), si.readByte()); } @@ -176,6 +177,7 @@ public class PagedBytesReferenceTests extends ESTestCase { // bulk-read all si.readFully(targetBytes); assertArrayEquals(pbrBytesWithOffset, targetBytes); + assertEquals(si.available(), 0); } public void testRandomReads() throws IOException { @@ -216,18 +218,22 @@ public class PagedBytesReferenceTests extends ESTestCase { int sliceLength = length - sliceOffset; BytesReference slice = pbr.slice(sliceOffset, sliceLength); StreamInput sliceInput = slice.streamInput(); + assertEquals(sliceInput.available(), sliceLength); // single reads assertEquals(slice.get(0), sliceInput.readByte()); assertEquals(slice.get(1), sliceInput.readByte()); assertEquals(slice.get(2), sliceInput.readByte()); + assertEquals(sliceInput.available(), sliceLength - 3); // reset the slice stream for bulk reading sliceInput.reset(); + assertEquals(sliceInput.available(), sliceLength); // bulk read byte[] sliceBytes = new byte[sliceLength]; sliceInput.readFully(sliceBytes); + assertEquals(sliceInput.available(), 0); // compare slice content with upper half of original byte[] pbrSliceBytes = Arrays.copyOfRange(pbr.toBytes(), sliceOffset, length); @@ -239,11 +245,14 @@ public class PagedBytesReferenceTests extends ESTestCase { assertArrayEquals(sliceBytes, sliceToBytes); sliceInput.reset(); + assertEquals(sliceInput.available(), sliceLength); byte[] buffer = new byte[sliceLength + scaledRandomIntBetween(1, 100)]; int offset = scaledRandomIntBetween(0, Math.max(1, buffer.length - sliceLength - 1)); int read = sliceInput.read(buffer, offset, sliceLength / 2); + assertEquals(sliceInput.available(), sliceLength - read); sliceInput.read(buffer, offset + read, sliceLength); assertArrayEquals(sliceBytes, Arrays.copyOfRange(buffer, offset, offset + sliceLength)); + assertEquals(sliceInput.available(), 0); } public void testWriteToOutputStream() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java index 7f232363f73..80bad2e1ecc 100644 --- a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java @@ -273,7 +273,9 @@ public class BytesStreamsTests extends ESTestCase { out.writeString("hello"); out.writeString("goodbye"); out.writeGenericValue(BytesRefs.toBytesRef("bytesref")); + final byte[] bytes = out.bytes().toBytes(); StreamInput in = StreamInput.wrap(out.bytes().toBytes()); + assertEquals(in.available(), bytes.length); assertThat(in.readBoolean(), equalTo(false)); assertThat(in.readByte(), equalTo((byte)1)); assertThat(in.readShort(), equalTo((short)-1)); @@ -302,9 +304,12 @@ public class BytesStreamsTests extends ESTestCase { namedWriteableRegistry.registerPrototype(BaseNamedWriteable.class, new TestNamedWriteable(null, null)); TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); out.writeNamedWriteable(namedWriteableIn); - StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes().toBytes()), namedWriteableRegistry); + byte[] bytes = out.bytes().toBytes(); + StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry); + assertEquals(in.available(), bytes.length); BaseNamedWriteable namedWriteableOut = in.readNamedWriteable(BaseNamedWriteable.class); assertEquals(namedWriteableOut, namedWriteableIn); + assertEquals(in.available(), 0); } public void testNamedWriteableDuplicates() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java index b5f26dba8a5..72f933462e0 100644 --- a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java @@ -23,7 +23,10 @@ import org.elasticsearch.common.bytes.ByteBufferBytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.test.ESTestCase; +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -93,4 +96,29 @@ public class StreamTests extends ESTestCase { index++; } } + + public void testFilterStreamInputDelegatesAvailable() throws IOException { + final int length = randomIntBetween(1, 1024); + StreamInput delegate = StreamInput.wrap(new byte[length]); + + FilterStreamInput filterInputStream = new FilterStreamInput(delegate) {}; + assertEquals(filterInputStream.available(), length); + + // read some bytes + final int bytesToRead = randomIntBetween(1, length); + filterInputStream.readBytes(new byte[bytesToRead], 0, bytesToRead); + assertEquals(filterInputStream.available(), length - bytesToRead); + } + + public void testInputStreamStreamInputDelegatesAvailable() throws IOException { + final int length = randomIntBetween(1, 1024); + ByteArrayInputStream is = new ByteArrayInputStream(new byte[length]); + InputStreamStreamInput streamInput = new InputStreamStreamInput(is); + assertEquals(streamInput.available(), length); + + // read some bytes + final int bytesToRead = randomIntBetween(1, length); + streamInput.readBytes(new byte[bytesToRead], 0, bytesToRead); + assertEquals(streamInput.available(), length - bytesToRead); + } } From e6eefcb1429e66dc23b49267e04bba1b4d33ec14 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 21 Mar 2016 10:16:02 -0400 Subject: [PATCH 05/45] Jimfs throws IAE when it should throw UOE This commit adds a hack to detect when Jimfs throws an IAE where it should be throwing an UOE. Namely, the method FileSystemProvider#createDirectory should be throwing an UOE if an attempt is made to set attributes that the filesystem does not support, but instead Jimfs violates this and throws an IAE. --- .../plugins/InstallPluginCommand.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index e822021e0b6..ce589acf844 100644 --- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -56,7 +56,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -307,11 +306,28 @@ class InstallPluginCommand extends Command { perms.add(PosixFilePermission.OTHERS_READ); perms.add(PosixFilePermission.OTHERS_EXECUTE); return Files.createTempDirectory(pluginsDir, ".installing-", PosixFilePermissions.asFileAttribute(perms)); - } catch (IllegalArgumentException | UnsupportedOperationException e) { - return Files.createTempDirectory(pluginsDir, ".installing-"); + } catch (IllegalArgumentException e) { + // Jimfs throws an IAE where it should throw an UOE + // remove when google/jimfs#30 is integrated into Jimfs + // and the Jimfs test dependency is upgraded to include + // this pull request + final StackTraceElement[] elements = e.getStackTrace(); + if (elements.length >= 1 && + elements[0].getClassName().equals("com.google.common.jimfs.AttributeService") && + elements[0].getMethodName().equals("setAttributeInternal")) { + return stagingDirectoryWithoutPosixPermissions(pluginsDir); + } else { + throw e; + } + } catch (UnsupportedOperationException e) { + return stagingDirectoryWithoutPosixPermissions(pluginsDir); } } + private Path stagingDirectoryWithoutPosixPermissions(Path pluginsDir) throws IOException { + return Files.createTempDirectory(pluginsDir, ".installing-"); + } + /** Load information about the plugin, and verify it can be installed with no errors. */ private PluginInfo verify(Terminal terminal, Path pluginRoot, boolean isBatch) throws Exception { // read and validate the plugin descriptor From 5dc48e71d04ff749a697df96023ebf9352ce91c6 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 22 Mar 2016 09:48:06 -0400 Subject: [PATCH 06/45] Use mock filesystem during install plugins tests This commit sets up the default filesystem used during install plugins tests. A hack is neeeded to handle the temporary directory because the system property "java.io.tmpdir" will have been initialized to a value that is sensible for the default filesystem, but not necessarily to a value that makes sense for the mock filesystem in use during the tests. This property is restored after each test. --- .../elasticsearch/common/io/PathUtils.java | 36 +++++-------------- .../common/settings/Setting.java | 1 - .../org/elasticsearch/env/Environment.java | 15 +++----- .../InternalSettingsPreparerTests.java | 1 - .../plugins/InstallPluginCommandTests.java | 34 +++++++++++++----- 5 files changed, 38 insertions(+), 49 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/io/PathUtils.java b/core/src/main/java/org/elasticsearch/common/io/PathUtils.java index ab5b46f3596..0002c1e5b5f 100644 --- a/core/src/main/java/org/elasticsearch/common/io/PathUtils.java +++ b/core/src/main/java/org/elasticsearch/common/io/PathUtils.java @@ -27,7 +27,7 @@ import java.nio.file.FileSystems; import java.nio.file.Path; import java.nio.file.Paths; -/** +/** * Utilities for creating a Path from names, * or accessing the default FileSystem. *

@@ -39,14 +39,14 @@ import java.nio.file.Paths; public final class PathUtils { /** no instantiation */ private PathUtils() {} - + /** the actual JDK default */ static final FileSystem ACTUAL_DEFAULT = FileSystems.getDefault(); - + /** can be changed by tests */ static volatile FileSystem DEFAULT = ACTUAL_DEFAULT; - - /** + + /** * Returns a {@code Path} from name components. *

* This works just like {@code Paths.get()}. @@ -57,30 +57,10 @@ public final class PathUtils { * a path against an existing one! */ public static Path get(String first, String... more) { - return get(DEFAULT, first, more); + return DEFAULT.getPath(first, more); } - - /** - * Returns a {@code Path} from name components against the given - * {@code FileSystem}. - *

- * This works just like {@code Paths.get()}. - * Remember: just like {@code Paths.get()} this is NOT A STRING CONCATENATION - * UTILITY FUNCTION. - *

- * Remember: this should almost never be used. Usually resolve - * a path against an existing one! - * - * @param fs the given {@code FileSystem} - * @param first the first path component - * @param more the remaining path components - * @return a path - */ - public static Path get(FileSystem fs, String first, String... more) { - return fs.getPath(first, more); - } - - /** + + /** * Returns a {@code Path} from a URI *

* This works just like {@code Paths.get()}. diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java index e600f845d86..f0e1b2e64ea 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -38,7 +38,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; -import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; diff --git a/core/src/main/java/org/elasticsearch/env/Environment.java b/core/src/main/java/org/elasticsearch/env/Environment.java index ca7363a32ee..e022ce6ad2f 100644 --- a/core/src/main/java/org/elasticsearch/env/Environment.java +++ b/core/src/main/java/org/elasticsearch/env/Environment.java @@ -31,7 +31,6 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.nio.file.FileStore; -import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -109,32 +108,28 @@ public class Environment { } public Environment(Settings settings) { - this(PathUtils.getDefaultFileSystem(), settings); - } - - public Environment(FileSystem fs, Settings settings) { this.settings = settings; final Path homeFile; if (PATH_HOME_SETTING.exists(settings)) { - homeFile = PathUtils.get(fs, PATH_HOME_SETTING.get(settings)); + homeFile = PathUtils.get(cleanPath(PATH_HOME_SETTING.get(settings))); } else { throw new IllegalStateException(PATH_HOME_SETTING.getKey() + " is not configured"); } if (PATH_CONF_SETTING.exists(settings)) { - configFile = PathUtils.get(fs, cleanPath(PATH_CONF_SETTING.get(settings))); + configFile = PathUtils.get(cleanPath(PATH_CONF_SETTING.get(settings))); } else { configFile = homeFile.resolve("config"); } if (PATH_SCRIPTS_SETTING.exists(settings)) { - scriptsFile = PathUtils.get(fs, cleanPath(PATH_SCRIPTS_SETTING.get(settings))); + scriptsFile = PathUtils.get(cleanPath(PATH_SCRIPTS_SETTING.get(settings))); } else { scriptsFile = configFile.resolve("scripts"); } if (PATH_PLUGINS_SETTING.exists(settings)) { - pluginsFile = PathUtils.get(fs, cleanPath(PATH_PLUGINS_SETTING.get(settings))); + pluginsFile = PathUtils.get(cleanPath(PATH_PLUGINS_SETTING.get(settings))); } else { pluginsFile = homeFile.resolve("plugins"); } @@ -144,7 +139,7 @@ public class Environment { dataFiles = new Path[dataPaths.size()]; dataWithClusterFiles = new Path[dataPaths.size()]; for (int i = 0; i < dataPaths.size(); i++) { - dataFiles[i] = PathUtils.get(fs, dataPaths.get(i)); + dataFiles[i] = PathUtils.get(dataPaths.get(i)); dataWithClusterFiles[i] = dataFiles[i].resolve(ClusterName.clusterNameFromSettings(settings).value()); } } else { diff --git a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java index 8f12ebc4597..c979b2f4013 100644 --- a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java +++ b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java @@ -21,7 +21,6 @@ package org.elasticsearch.node.internal; import java.io.IOException; import java.io.InputStream; -import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index 7c516863637..fdcfc44e989 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -23,14 +23,17 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.google.common.jimfs.Configuration; import com.google.common.jimfs.Jimfs; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.SuppressForbidden; import org.elasticsearch.Version; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cli.UserError; import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.io.PathUtilsForTesting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.PosixPermissionsResetter; +import org.junit.After; import java.io.IOException; import java.io.InputStream; @@ -71,12 +74,25 @@ public class InstallPluginCommandTests extends ESTestCase { private final FileSystem fs; private final boolean isPosix; private final boolean isReal; + private final String javaIoTmpdir; + @SuppressForbidden(reason = "sets java.io.tmpdir") public InstallPluginCommandTests(FileSystem fs, Function temp) { this.fs = fs; this.temp = temp; this.isPosix = fs.supportedFileAttributeViews().contains("posix"); this.isReal = fs == PathUtils.getDefaultFileSystem(); + PathUtilsForTesting.installMock(fs); + javaIoTmpdir = System.getProperty("java.io.tmpdir"); + System.setProperty("java.io.tmpdir", temp.apply("tmpdir").toString()); + } + + @After + @SuppressForbidden(reason = "resets java.io.tmpdir") + public void tearDown() throws Exception { + System.setProperty("java.io.tmpdir", javaIoTmpdir); + PathUtilsForTesting.teardown(); + super.tearDown(); } @ParametersFactory @@ -85,26 +101,26 @@ public class InstallPluginCommandTests extends ESTestCase { private final FileSystem fileSystem; private final Function temp; - public Parameter(FileSystem fileSystem, Supplier root) { - this(fileSystem, root, s -> { + public Parameter(FileSystem fileSystem, String root) { + this(fileSystem, s -> { try { - return Files.createTempDirectory(fileSystem.getPath(root.get()), s); + return Files.createTempDirectory(fileSystem.getPath(root), s); } catch (IOException e) { throw new RuntimeException(e); } }); } - public Parameter(FileSystem fileSystem, Supplier root, Function temp) { + public Parameter(FileSystem fileSystem, Function temp) { this.fileSystem = fileSystem; this.temp = temp; } } List parameters = new ArrayList<>(); - parameters.add(new Parameter(Jimfs.newFileSystem(Configuration.windows()), () -> "c:\\")); - parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.osX())), () -> "/")); - parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.unix())), () -> "/")); - parameters.add(new Parameter(PathUtils.getDefaultFileSystem(), () -> createTempDir().toString(), LuceneTestCase::createTempDir )); + parameters.add(new Parameter(Jimfs.newFileSystem(Configuration.windows()), "c:\\")); + parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.osX())), "/")); + parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.unix())), "/")); + parameters.add(new Parameter(PathUtils.getDefaultFileSystem(), LuceneTestCase::createTempDir )); return parameters.stream().map(p -> new Object[] { p.fileSystem, p.temp }).collect(Collectors.toList()); } @@ -124,7 +140,7 @@ public class InstallPluginCommandTests extends ESTestCase { Settings settings = Settings.builder() .put("path.home", home) .build(); - return new Environment(fs, settings); + return new Environment(settings); } static Path createPluginDir(Function temp) throws IOException { From fdac0c7c6cc521ed10c3a20f1533769a4041665e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Mar 2016 12:07:57 -0700 Subject: [PATCH 07/45] Link to named queries docs from bool query page The named queries feature only makes sense with bool queries, but was not cross-referenced from the bool query documentation page. --- docs/reference/query-dsl/bool-query.asciidoc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc index f8e08aaada9..dc986bf14d9 100644 --- a/docs/reference/query-dsl/bool-query.asciidoc +++ b/docs/reference/query-dsl/bool-query.asciidoc @@ -141,3 +141,10 @@ GET _search } --------------------------------- // AUTOSENSE + +==== Using named queries to see which clauses matched + +If you need to know which of the clauses in the bool query matched the documents +returned from the query, you can use +<> to assign a name to +each clause. From e907b7c11e5ed4b0ed9ed3bb4a04b3f565bfe854 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 23 Mar 2016 10:39:54 +0100 Subject: [PATCH 08/45] Check that S3 setting `buffer_size` is always lower than `chunk_size` We can be better at checking `buffer_size` and `chunk_size` for S3 repositories. For example, we know that: * `buffer_size` should be more than `5mb` * `chunk_size` should be no more than `5tb` * `buffer_size` should be lower than `chunk_size` Otherwise, setting `buffer_size` is useless. For the record: `chunk_size` is a Snapshot setting whatever the implementation is. `buffer_size` is an S3 implementation setting. Let say that you are snapshotting a 500mb file. If you set `chunk_size` to `200mb`, then Snapshot service will call S3 repository to snapshot 3 files with the following sizes: * `200mb` * `200mb` * `100mb` If you set `buffer_size` to `100mb` (AWS maximum size recommendation), the first file of `200mb` will be uploaded on S3 using the multipart feature in 2 chunks and the workflow is basically the following: * create the multipart request and get back an `id` from AWS S3 platform * upload part1: `100mb` * upload part2: `100mb` * "commit" the full upload using the `id`. Closes #17244. --- .../common/settings/Setting.java | 22 +++++++ docs/plugins/repository-s3.asciidoc | 4 +- .../cloud/aws/blobstore/S3BlobStore.java | 10 +--- .../repositories/s3/S3Repository.java | 23 ++++++-- .../cloud/aws/RepositoryS3SettingsTests.java | 59 +++++++++++++++++-- .../aws/blobstore/S3OutputStreamTests.java | 4 +- 6 files changed, 100 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java index f0e1b2e64ea..a2ebe7a2c30 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -523,6 +523,28 @@ public class Setting extends ToXContentToBytes { return new Setting<>(key, defaultValue, (s) -> ByteSizeValue.parseBytesSizeValue(s, key), properties); } + public static Setting byteSizeSetting(String key, ByteSizeValue value, ByteSizeValue minValue, ByteSizeValue maxValue, + Property... properties) { + return byteSizeSetting(key, (s) -> value.toString(), minValue, maxValue, properties); + } + + public static Setting byteSizeSetting(String key, Function defaultValue, + ByteSizeValue minValue, ByteSizeValue maxValue, + Property... properties) { + return new Setting<>(key, defaultValue, (s) -> parseByteSize(s, minValue, maxValue, key), properties); + } + + public static ByteSizeValue parseByteSize(String s, ByteSizeValue minValue, ByteSizeValue maxValue, String key) { + ByteSizeValue value = ByteSizeValue.parseBytesSizeValue(s, key); + if (value.bytes() < minValue.bytes()) { + throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be >= " + minValue); + } + if (value.bytes() > maxValue.bytes()) { + throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be =< " + maxValue); + } + return value; + } + public static Setting positiveTimeSetting(String key, TimeValue defaultValue, Property... properties) { return timeSetting(key, defaultValue, TimeValue.timeValueMillis(0), properties); } diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 007018a6abc..8780b3a710e 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -189,7 +189,7 @@ The following settings are supported: Big files can be broken down into chunks during snapshotting if needed. The chunk size can be specified in bytes or by using size value notation, - i.e. `1g`, `10m`, `5k`. Defaults to `100m`. + i.e. `1gb`, `10mb`, `5kb`. Defaults to `1gb`. `compress`:: @@ -210,7 +210,7 @@ The following settings are supported: to split the chunk into several parts, each of `buffer_size` length, and to upload each part in its own request. Note that setting a buffer size lower than `5mb` is not allowed since it will prevents the use of the - Multipart API and may result in upload errors. Defaults to `5mb`. + Multipart API and may result in upload errors. Defaults to `100mb`. `max_retries`:: diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java index a8bf3ea2959..650d71f62ad 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java @@ -36,7 +36,6 @@ import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.BlobStoreException; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import java.util.ArrayList; @@ -47,8 +46,6 @@ import java.util.Locale; */ public class S3BlobStore extends AbstractComponent implements BlobStore { - public static final ByteSizeValue MIN_BUFFER_SIZE = new ByteSizeValue(5, ByteSizeUnit.MB); - private final AmazonS3 client; private final String bucket; @@ -72,12 +69,7 @@ public class S3BlobStore extends AbstractComponent implements BlobStore { this.bucket = bucket; this.region = region; this.serverSideEncryption = serverSideEncryption; - - this.bufferSize = (bufferSize != null) ? bufferSize : MIN_BUFFER_SIZE; - if (this.bufferSize.getBytes() < MIN_BUFFER_SIZE.getBytes()) { - throw new BlobStoreException("Detected a buffer_size for the S3 storage lower than [" + MIN_BUFFER_SIZE + "]"); - } - + this.bufferSize = bufferSize; this.cannedACL = initCannedACL(cannedACL); this.numberOfRetries = maxRetries; this.storageClass = initStorageClass(storageClass); diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index fde774a6b92..6742233cf78 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -106,19 +106,21 @@ public class S3Repository extends BlobStoreRepository { * repositories.s3.buffer_size: Minimum threshold below which the chunk is uploaded using a single request. Beyond this threshold, * the S3 repository will use the AWS Multipart Upload API to split the chunk into several parts, each of buffer_size length, and * to upload each part in its own request. Note that setting a buffer size lower than 5mb is not allowed since it will prevents the - * use of the Multipart API and may result in upload errors. Defaults to 5mb. + * use of the Multipart API and may result in upload errors. Defaults to 100m. */ Setting BUFFER_SIZE_SETTING = - Setting.byteSizeSetting("repositories.s3.buffer_size", S3BlobStore.MIN_BUFFER_SIZE, Property.NodeScope); + Setting.byteSizeSetting("repositories.s3.buffer_size", new ByteSizeValue(100, ByteSizeUnit.MB), + new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope); /** * repositories.s3.max_retries: Number of retries in case of S3 errors. Defaults to 3. */ Setting MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, Property.NodeScope); /** - * repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 100m. + * repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g. */ Setting CHUNK_SIZE_SETTING = - Setting.byteSizeSetting("repositories.s3.chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB), Property.NodeScope); + Setting.byteSizeSetting("repositories.s3.chunk_size", new ByteSizeValue(1, ByteSizeUnit.GB), + new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope); /** * repositories.s3.compress: When set to true metadata files are stored in compressed format. This setting doesn’t affect index * files that are already compressed by default. Defaults to false. @@ -187,7 +189,8 @@ public class S3Repository extends BlobStoreRepository { * @see Repositories#BUFFER_SIZE_SETTING */ Setting BUFFER_SIZE_SETTING = - Setting.byteSizeSetting("buffer_size", S3BlobStore.MIN_BUFFER_SIZE, Property.NodeScope); + Setting.byteSizeSetting("buffer_size", new ByteSizeValue(100, ByteSizeUnit.MB), + new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope); /** * max_retries * @see Repositories#MAX_RETRIES_SETTING @@ -197,7 +200,9 @@ public class S3Repository extends BlobStoreRepository { * chunk_size * @see Repositories#CHUNK_SIZE_SETTING */ - Setting CHUNK_SIZE_SETTING = Setting.byteSizeSetting("chunk_size", "-1", Property.NodeScope); + Setting CHUNK_SIZE_SETTING = + Setting.byteSizeSetting("chunk_size", new ByteSizeValue(1, ByteSizeUnit.GB), + new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope); /** * compress * @see Repositories#COMPRESS_SETTING @@ -260,6 +265,12 @@ public class S3Repository extends BlobStoreRepository { this.chunkSize = getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING); this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING); + // We make sure that chunkSize is bigger or equal than/to bufferSize + if (this.chunkSize.getMb() < bufferSize.getMb()) { + throw new RepositoryException(name.name(), Repository.CHUNK_SIZE_SETTING.getKey() + " (" + this.chunkSize + + ") can't be lower than " + Repository.BUFFER_SIZE_SETTING.getKey() + " (" + bufferSize + ")."); + } + // Parse and validate the user's S3 Storage Class setting String storageClass = getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING); String cannedACL = getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java index 7d881e0dd30..c5c617e8591 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java @@ -21,12 +21,20 @@ package org.elasticsearch.cloud.aws; import com.amazonaws.Protocol; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.repositories.RepositoryException; +import org.elasticsearch.repositories.RepositoryName; import org.elasticsearch.repositories.RepositorySettings; +import org.elasticsearch.repositories.s3.S3Repository; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; + import static org.elasticsearch.repositories.s3.S3Repository.Repositories; import static org.elasticsearch.repositories.s3.S3Repository.Repository; import static org.elasticsearch.repositories.s3.S3Repository.getValue; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isEmptyString; @@ -110,9 +118,9 @@ public class RepositoryS3SettingsTests extends ESTestCase { assertThat(AwsS3Service.CLOUD_S3.SIGNER_SETTING.get(nodeSettings), is("global-signer")); assertThat(getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING), is(false)); - assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(5L)); + assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(100L)); assertThat(getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING), is(3)); - assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getMb(), is(100L)); + assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getGb(), is(1L)); assertThat(getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING), is(false)); assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString()); assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString()); @@ -138,9 +146,9 @@ public class RepositoryS3SettingsTests extends ESTestCase { assertThat(AwsS3Service.CLOUD_S3.SIGNER_SETTING.get(nodeSettings), is("s3-signer")); assertThat(getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING), is(false)); - assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(5L)); + assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(100L)); assertThat(getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING), is(3)); - assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getMb(), is(100L)); + assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getGb(), is(1L)); assertThat(getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING), is(false)); assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString()); assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString()); @@ -292,6 +300,25 @@ public class RepositoryS3SettingsTests extends ESTestCase { assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), is("repository-basepath")); } + /** + * We test wrong Chunk and Buffer settings + */ + public void testInvalidChunkBufferSizeRepositorySettings() throws IOException { + // chunk < buffer should fail + internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.MB), + "chunk_size (5mb) can't be lower than buffer_size (10mb)."); + // chunk > buffer should pass + internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(10, ByteSizeUnit.MB), null); + // chunk = buffer should pass + internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.MB), null); + // buffer < 5mb should fail + internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(4, ByteSizeUnit.MB), new ByteSizeValue(10, ByteSizeUnit.MB), + "Failed to parse value [4mb] for setting [buffer_size] must be >= 5mb"); + // chunk > 5tb should fail + internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(6, ByteSizeUnit.TB), + "Failed to parse value [6tb] for setting [chunk_size] must be =< 5tb"); + } + private Settings buildSettings(Settings... global) { Settings.Builder builder = Settings.builder(); for (Settings settings : global) { @@ -299,4 +326,28 @@ public class RepositoryS3SettingsTests extends ESTestCase { } return builder.build(); } + + private void internalTestInvalidChunkBufferSizeSettings(ByteSizeValue buffer, ByteSizeValue chunk, String expectedMessage) + throws IOException { + Settings nodeSettings = buildSettings(AWS, S3, REPOSITORIES); + RepositorySettings s3RepositorySettings = new RepositorySettings(nodeSettings, Settings.builder() + .put(Repository.BUFFER_SIZE_SETTING.getKey(), buffer) + .put(Repository.CHUNK_SIZE_SETTING.getKey(), chunk) + .build()); + + try { + new S3Repository(new RepositoryName("s3", "s3repo"), s3RepositorySettings, null, null); + fail("We should either raise a NPE or a RepositoryException or a IllegalArgumentException"); + } catch (RepositoryException e) { + assertThat(e.getDetailedMessage(), containsString(expectedMessage)); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString(expectedMessage)); + } catch (NullPointerException e) { + // Because we passed to the CTOR a Null AwsS3Service, we get a NPE which is expected + // in the context of this test + if (expectedMessage != null) { + fail("We should have raised a RepositoryException"); + } + } + } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java index f023b64211f..0518d657e23 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java @@ -19,6 +19,8 @@ package org.elasticsearch.cloud.aws.blobstore; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.test.ESTestCase; import java.io.ByteArrayOutputStream; @@ -33,7 +35,7 @@ import static org.hamcrest.Matchers.is; * Unit test for {@link S3OutputStream}. */ public class S3OutputStreamTests extends ESTestCase { - private static final int BUFFER_SIZE = S3BlobStore.MIN_BUFFER_SIZE.bytesAsInt(); + private static final int BUFFER_SIZE = new ByteSizeValue(5, ByteSizeUnit.MB).bytesAsInt(); public void testWriteLessDataThanBufferSize() throws IOException { MockDefaultS3OutputStream out = newS3OutputStream(BUFFER_SIZE); From b614e3e075b8cba168b00a071023ab313cb61fb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 22 Mar 2016 18:25:22 +0100 Subject: [PATCH 09/45] Switch to using refactored SortBuilder in SearchSourceBuilder and elsewhere Switching from using list of BytesReference to real SortBuilder list in SearchSourceBuilder, TopHitsAggregatorBuilder and TopHitsAggregatorFactory. Removing SortParseElement and related sort parsers. --- .../common/io/stream/StreamInput.java | 19 +- .../common/io/stream/StreamOutput.java | 25 +- .../support/InnerHitsQueryParserHelper.java | 11 +- .../support/NestedInnerQueryParseSupport.java | 134 ---------- .../elasticsearch/search/SearchModule.java | 13 + .../elasticsearch/search/SearchService.java | 34 +-- .../tophits/TopHitsAggregatorBuilder.java | 52 ++-- .../tophits/TopHitsAggregatorFactory.java | 41 +-- .../metrics/tophits/TopHitsParser.java | 15 +- .../search/builder/SearchSourceBuilder.java | 49 ++-- .../innerhits/InnerHitsFetchSubPhase.java | 5 +- .../innerhits/InnerHitsParseElement.java | 9 +- .../search/query/QueryPhase.java | 2 - .../search/sort/FieldSortBuilder.java | 21 +- .../search/sort/GeoDistanceSortBuilder.java | 2 +- .../search/sort/GeoDistanceSortParser.java | 220 --------------- .../search/sort/ScoreSortBuilder.java | 2 +- .../search/sort/ScriptSortBuilder.java | 2 +- .../search/sort/ScriptSortParser.java | 210 --------------- .../search/sort/SortBuilder.java | 38 +++ .../search/sort/SortParseElement.java | 252 ------------------ .../elasticsearch/search/sort/SortParser.java | 36 --- .../builder/SearchSourceBuilderTests.java | 14 +- .../search/sort/AbstractSortTestCase.java | 20 +- .../search/sort/FieldSortBuilderTests.java | 19 +- .../sort/GeoDistanceSortBuilderTests.java | 8 + .../search/sort/ScoreSortBuilderTests.java | 7 + .../search/sort/ScriptSortBuilderTests.java | 11 +- .../search/sort/SortBuilderTests.java | 4 +- .../search/sort/SortParserTests.java | 13 +- 30 files changed, 221 insertions(+), 1067 deletions(-) delete mode 100644 core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java delete mode 100644 core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java delete mode 100644 core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java delete mode 100644 core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java delete mode 100644 core/src/main/java/org/elasticsearch/search/sort/SortParser.java diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index aca136a2a9a..1fa96a0f2c2 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -37,14 +37,13 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.text.Text; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; -import org.elasticsearch.ingest.IngestStats; -import org.elasticsearch.search.rescore.RescoreBuilder; -import org.elasticsearch.search.suggest.SuggestionBuilder; -import org.elasticsearch.search.suggest.completion.context.QueryContext; -import org.elasticsearch.search.suggest.phrase.SmoothingModel; -import org.elasticsearch.tasks.Task; import org.elasticsearch.search.aggregations.AggregatorBuilder; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder; +import org.elasticsearch.search.rescore.RescoreBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.suggest.SuggestionBuilder; +import org.elasticsearch.search.suggest.phrase.SmoothingModel; +import org.elasticsearch.tasks.Task; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -68,7 +67,6 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; import java.util.function.Supplier; import static org.elasticsearch.ElasticsearchException.readException; @@ -726,6 +724,13 @@ public abstract class StreamInput extends InputStream { return readNamedWriteable(SuggestionBuilder.class); } + /** + * Reads a {@link SortBuilder} from the current stream + */ + public SortBuilder readSortBuilder() throws IOException { + return readNamedWriteable(SortBuilder.class); + } + /** * Reads a {@link org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder} from the current stream */ diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index dd357e27709..630c297edf6 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -36,13 +36,13 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.text.Text; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; -import org.elasticsearch.search.rescore.RescoreBuilder; -import org.elasticsearch.search.suggest.SuggestionBuilder; -import org.elasticsearch.search.suggest.completion.context.QueryContext; -import org.elasticsearch.search.suggest.phrase.SmoothingModel; -import org.elasticsearch.tasks.Task; import org.elasticsearch.search.aggregations.AggregatorBuilder; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder; +import org.elasticsearch.search.rescore.RescoreBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.suggest.SuggestionBuilder; +import org.elasticsearch.search.suggest.phrase.SmoothingModel; +import org.elasticsearch.tasks.Task; import org.joda.time.ReadableInstant; import java.io.EOFException; @@ -532,7 +532,7 @@ public abstract class StreamOutput extends OutputStream { } } - public void writeOptionalWriteable(@Nullable Writeable writeable) throws IOException { + public void writeOptionalWriteable(@Nullable Writeable writeable) throws IOException { if (writeable != null) { writeBoolean(true); writeable.writeTo(this); @@ -663,7 +663,7 @@ public abstract class StreamOutput extends OutputStream { /** * Writes a {@link NamedWriteable} to the current stream, by first writing its name and then the object itself */ - void writeNamedWriteable(NamedWriteable namedWriteable) throws IOException { + void writeNamedWriteable(NamedWriteable namedWriteable) throws IOException { writeString(namedWriteable.getWriteableName()); namedWriteable.writeTo(this); } @@ -685,7 +685,7 @@ public abstract class StreamOutput extends OutputStream { /** * Writes a {@link QueryBuilder} to the current stream */ - public void writeQuery(QueryBuilder queryBuilder) throws IOException { + public void writeQuery(QueryBuilder queryBuilder) throws IOException { writeNamedWriteable(queryBuilder); } @@ -745,8 +745,15 @@ public abstract class StreamOutput extends OutputStream { /** * Writes a {@link SuggestionBuilder} to the current stream */ - public void writeSuggestion(SuggestionBuilder suggestion) throws IOException { + public void writeSuggestion(SuggestionBuilder suggestion) throws IOException { writeNamedWriteable(suggestion); } + /** + * Writes a {@link SortBuilder} to the current stream + */ + public void writeSortBuilder(SortBuilder sort) throws IOException { + writeNamedWriteable(sort); + } + } diff --git a/core/src/main/java/org/elasticsearch/index/query/support/InnerHitsQueryParserHelper.java b/core/src/main/java/org/elasticsearch/index/query/support/InnerHitsQueryParserHelper.java index 74a905919f7..0467e459718 100644 --- a/core/src/main/java/org/elasticsearch/index/query/support/InnerHitsQueryParserHelper.java +++ b/core/src/main/java/org/elasticsearch/index/query/support/InnerHitsQueryParserHelper.java @@ -27,7 +27,7 @@ import org.elasticsearch.search.fetch.source.FetchSourceParseElement; import org.elasticsearch.search.highlight.HighlighterParseElement; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.SubSearchContext; -import org.elasticsearch.search.sort.SortParseElement; +import org.elasticsearch.search.sort.SortBuilder; import java.io.IOException; @@ -35,7 +35,6 @@ public class InnerHitsQueryParserHelper { public static final InnerHitsQueryParserHelper INSTANCE = new InnerHitsQueryParserHelper(); - private static final SortParseElement sortParseElement = new SortParseElement(); private static final FetchSourceParseElement sourceParseElement = new FetchSourceParseElement(); private static final HighlighterParseElement highlighterParseElement = new HighlighterParseElement(); private static final ScriptFieldsParseElement scriptFieldsParseElement = new ScriptFieldsParseElement(); @@ -54,10 +53,10 @@ public class InnerHitsQueryParserHelper { if ("name".equals(fieldName)) { innerHitName = parser.textOrNull(); } else { - parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sortParseElement, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement); + parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement); } } else { - parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sortParseElement, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement); + parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement); } } } catch (Exception e) { @@ -67,10 +66,10 @@ public class InnerHitsQueryParserHelper { } public static void parseCommonInnerHitOptions(XContentParser parser, XContentParser.Token token, String fieldName, SubSearchContext subSearchContext, - SortParseElement sortParseElement, FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement, + FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement, ScriptFieldsParseElement scriptFieldsParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement) throws Exception { if ("sort".equals(fieldName)) { - sortParseElement.parse(parser, subSearchContext); + SortBuilder.parseSort(parser, subSearchContext); } else if ("_source".equals(fieldName)) { sourceParseElement.parse(parser, subSearchContext); } else if (token == XContentParser.Token.START_OBJECT) { diff --git a/core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java b/core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java deleted file mode 100644 index 5c65a57d532..00000000000 --- a/core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.query.support; - -import org.apache.lucene.search.Query; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.object.ObjectMapper; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryParseContext; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.QueryShardException; - -import java.io.IOException; - -/** - * A helper that helps with parsing inner queries of the nested query. - * 1) Takes into account that type nested path can appear before or after the inner query - * 2) Updates the {@link NestedScope} when parsing the inner query. - */ -public class NestedInnerQueryParseSupport { - - protected final QueryShardContext shardContext; - protected final QueryParseContext parseContext; - - private BytesReference source; - private Query innerFilter; - protected String path; - - private boolean filterParsed = false; - protected boolean filterFound = false; - - protected ObjectMapper nestedObjectMapper; - - public NestedInnerQueryParseSupport(XContentParser parser, QueryShardContext context) { - shardContext = context; - parseContext = shardContext.parseContext(); - shardContext.reset(parser); - } - - public void filter() throws IOException { - if (path != null) { - setPathLevel(); - try { - innerFilter = QueryBuilder.rewriteQuery(parseContext.parseInnerQueryBuilder(), - this.shardContext).toFilter(this.shardContext); - } finally { - resetPathLevel(); - } - filterParsed = true; - } else { - source = XContentFactory.smileBuilder().copyCurrentStructure(parseContext.parser()).bytes(); - } - filterFound = true; - } - - public Query getInnerFilter() throws IOException { - if (filterParsed) { - return innerFilter; - } else { - if (path == null) { - throw new QueryShardException(shardContext, "[nested] requires 'path' field"); - } - if (!filterFound) { - throw new QueryShardException(shardContext, "[nested] requires either 'query' or 'filter' field"); - } - - setPathLevel(); - XContentParser old = parseContext.parser(); - try { - XContentParser innerParser = XContentHelper.createParser(source); - parseContext.parser(innerParser); - innerFilter = QueryBuilder.rewriteQuery(parseContext.parseInnerQueryBuilder(), - this.shardContext).toFilter(this.shardContext); - filterParsed = true; - return innerFilter; - } finally { - resetPathLevel(); - parseContext.parser(old); - } - } - } - - public void setPath(String path) { - this.path = path; - nestedObjectMapper = shardContext.getObjectMapper(path); - if (nestedObjectMapper == null) { - throw new QueryShardException(shardContext, "[nested] failed to find nested object under path [" + path + "]"); - } - if (!nestedObjectMapper.nested().isNested()) { - throw new QueryShardException(shardContext, "[nested] nested object under path [" + path + "] is not of nested type"); - } - } - - public String getPath() { - return path; - } - - public ObjectMapper getNestedObjectMapper() { - return nestedObjectMapper; - } - - public boolean filterFound() { - return filterFound; - } - - private void setPathLevel() { - shardContext.nestedScope().nextLevel(nestedObjectMapper); - } - - private void resetPathLevel() { - shardContext.nestedScope().previousLevel(); - } - -} diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java index ae6d5aaf4be..96db4b11469 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java @@ -226,6 +226,11 @@ import org.elasticsearch.search.highlight.Highlighters; import org.elasticsearch.search.query.QueryPhase; import org.elasticsearch.search.rescore.QueryRescorerBuilder; import org.elasticsearch.search.rescore.RescoreBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.GeoDistanceSortBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.ScriptSortBuilder; +import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.suggest.Suggester; import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.search.suggest.SuggestionBuilder; @@ -346,6 +351,7 @@ public class SearchModule extends AbstractModule { configureFetchSubPhase(); configureShapes(); configureRescorers(); + configureSorts(); } protected void configureFetchSubPhase() { @@ -489,6 +495,13 @@ public class SearchModule extends AbstractModule { namedWriteableRegistry.registerPrototype(RescoreBuilder.class, QueryRescorerBuilder.PROTOTYPE); } + private void configureSorts() { + namedWriteableRegistry.registerPrototype(SortBuilder.class, GeoDistanceSortBuilder.PROTOTYPE); + namedWriteableRegistry.registerPrototype(SortBuilder.class, ScoreSortBuilder.PROTOTYPE); + namedWriteableRegistry.registerPrototype(SortBuilder.class, ScriptSortBuilder.PROTOTYPE); + namedWriteableRegistry.registerPrototype(SortBuilder.class, FieldSortBuilder.PROTOTYPE); + } + private void registerBuiltinFunctionScoreParsers() { registerFunctionScoreParser(new ScriptScoreFunctionParser()); registerFunctionScoreParser(new GaussDecayFunctionParser()); diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index a1b7f93d0d1..98085bfed35 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -21,6 +21,7 @@ package org.elasticsearch.search; import com.carrotsearch.hppc.ObjectFloatHashMap; import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.Sort; import org.apache.lucene.search.TopDocs; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.cache.recycler.PageCacheRecycler; @@ -41,7 +42,6 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.ConcurrentMapLong; import org.elasticsearch.common.util.concurrent.FutureUtils; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; @@ -92,6 +92,7 @@ import org.elasticsearch.search.query.QuerySearchResultProvider; import org.elasticsearch.search.query.ScrollQuerySearchResult; import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.searchafter.SearchAfterBuilder; +import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.threadpool.ThreadPool; @@ -99,6 +100,7 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicLong; @@ -683,33 +685,13 @@ public class SearchService extends AbstractLifecycleComponent imp context.parsedPostFilter(queryShardContext.toQuery(source.postFilter())); } if (source.sorts() != null) { - XContentParser completeSortParser = null; try { - XContentBuilder completeSortBuilder = XContentFactory.jsonBuilder(); - completeSortBuilder.startObject(); - completeSortBuilder.startArray("sort"); - for (BytesReference sort : source.sorts()) { - XContentParser parser = XContentFactory.xContent(sort).createParser(sort); - parser.nextToken(); - completeSortBuilder.copyCurrentStructure(parser); + Optional optionalSort = SortBuilder.buildSort(source.sorts(), context.getQueryShardContext()); + if (optionalSort.isPresent()) { + context.sort(optionalSort.get()); } - completeSortBuilder.endArray(); - completeSortBuilder.endObject(); - BytesReference completeSortBytes = completeSortBuilder.bytes(); - completeSortParser = XContentFactory.xContent(completeSortBytes).createParser(completeSortBytes); - completeSortParser.nextToken(); - completeSortParser.nextToken(); - completeSortParser.nextToken(); - this.elementParsers.get("sort").parse(completeSortParser, context); - } catch (Exception e) { - String sSource = "_na_"; - try { - sSource = source.toString(); - } catch (Throwable e1) { - // ignore - } - XContentLocation location = completeSortParser != null ? completeSortParser.getTokenLocation() : null; - throw new SearchParseException(context, "failed to parse sort source [" + sSource + "]", location, e); + } catch (IOException e) { + throw new SearchContextException(context, "failed to create sort elements", e); } } context.trackScores(source.trackScores()); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java index 9f5f3e443ef..279de65fcae 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java @@ -21,13 +21,9 @@ package org.elasticsearch.search.aggregations.metrics.tophits; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.script.Script; import org.elasticsearch.search.aggregations.AggregationInitializationException; import org.elasticsearch.search.aggregations.AggregatorBuilder; @@ -38,6 +34,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.HighlightBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; @@ -57,7 +54,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder sorts = null; + private List> sorts = null; private HighlightBuilder highlightBuilder; private List fieldNames; private List fieldDataFields; @@ -119,6 +116,9 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder sort) { if (sort == null) { throw new IllegalArgumentException("[sort] must not be null: [" + name + "]"); } - try { - if (sorts == null) { + if (sorts == null) { sorts = new ArrayList<>(); - } - // NORELEASE when sort has been refactored and made writeable - // add the sortBuilcer to the List directly instead of - // serialising to XContent - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - sort.toXContent(builder, EMPTY_PARAMS); - builder.endObject(); - sorts.add(builder.bytes()); - } catch (IOException e) { - throw new RuntimeException(e); } + sorts.add(sort); return this; } /** * Adds a sort builder. */ - public TopHitsAggregatorBuilder sorts(List sorts) { + public TopHitsAggregatorBuilder sorts(List> sorts) { if (sorts == null) { throw new IllegalArgumentException("[sorts] must not be null: [" + name + "]"); } if (this.sorts == null) { this.sorts = new ArrayList<>(); } - for (BytesReference sort : sorts) { + for (SortBuilder sort : sorts) { this.sorts.add(sort); } return this; @@ -181,7 +173,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder sorts() { + public List> sorts() { return sorts; } @@ -509,10 +501,10 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder sort : sorts) { + builder.startObject(); + sort.toXContent(builder, params); + builder.endObject(); } builder.endArray(); } @@ -562,9 +554,9 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder sorts = new ArrayList<>(); + List> sorts = new ArrayList<>(); for (int i = 0; i < size; i++) { - sorts.add(in.readBytesReference()); + sorts.add(in.readSortBuilder()); } factory.sorts = sorts; } @@ -612,8 +604,8 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder sort : sorts) { + out.writeSortBuilder(sort); } } out.writeBoolean(trackScores); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java index c4d2165e308..be10f4ce878 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java @@ -19,12 +19,7 @@ package org.elasticsearch.search.aggregations.metrics.tophits; -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.common.xcontent.XContentParser; +import org.apache.lucene.search.Sort; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.aggregations.Aggregator; @@ -35,27 +30,27 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext; -import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase; import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext.FieldDataField; +import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.internal.SubSearchContext; -import org.elasticsearch.search.sort.SortParseElement; +import org.elasticsearch.search.sort.SortBuilder; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; public class TopHitsAggregatorFactory extends AggregatorFactory { - private static final SortParseElement sortParseElement = new SortParseElement(); private final int from; private final int size; private final boolean explain; private final boolean version; private final boolean trackScores; - private final List sorts; + private final List> sorts; private final HighlightBuilder highlightBuilder; private final List fieldNames; private final List fieldDataFields; @@ -63,7 +58,7 @@ public class TopHitsAggregatorFactory extends AggregatorFactory sorts, HighlightBuilder highlightBuilder, List fieldNames, List fieldDataFields, + List> sorts, HighlightBuilder highlightBuilder, List fieldNames, List fieldDataFields, List scriptFields, FetchSourceContext fetchSourceContext, AggregationContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactories, Map metaData) throws IOException { super(name, type, context, parent, subFactories, metaData); @@ -90,27 +85,9 @@ public class TopHitsAggregatorFactory extends AggregatorFactory optionalSort = SortBuilder.buildSort(sorts, subSearchContext.getQueryShardContext()); + if (optionalSort.isPresent()) { + subSearchContext.sort(optionalSort.get()); } } if (fieldNames != null) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java index 03921d620ee..9ec6b46dafe 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java @@ -19,9 +19,6 @@ package org.elasticsearch.search.aggregations.metrics.tophits; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.script.Script; @@ -30,6 +27,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.HighlightBuilder; +import org.elasticsearch.search.sort.SortBuilder; + import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -124,9 +123,7 @@ public class TopHitsParser implements Aggregator.Parser { } else if (context.parseFieldMatcher().match(currentFieldName, SearchSourceBuilder.HIGHLIGHT_FIELD)) { factory.highlighter(HighlightBuilder.PROTOTYPE.fromXContent(context)); } else if (context.parseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SORT_FIELD)) { - List sorts = new ArrayList<>(); - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); - sorts.add(xContentBuilder.bytes()); + List> sorts = SortBuilder.fromXContent(context); factory.sorts(sorts); } else { throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", @@ -157,11 +154,7 @@ public class TopHitsParser implements Aggregator.Parser { } factory.fieldDataFields(fieldDataFields); } else if (context.parseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SORT_FIELD)) { - List sorts = new ArrayList<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); - sorts.add(xContentBuilder.bytes()); - } + List> sorts = SortBuilder.fromXContent(context); factory.sorts(sorts); } else if (context.parseFieldMatcher().match(currentFieldName, SearchSourceBuilder._SOURCE_FIELD)) { factory.fetchSource(FetchSourceContext.parse(parser, context)); diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index a20ec535238..67d25958310 100644 --- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -52,6 +52,7 @@ import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.searchafter.SearchAfterBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; @@ -139,7 +140,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ private Boolean version; - private List sorts; + private List> sorts; private boolean trackScores = false; @@ -336,6 +337,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ * The sort ordering */ public SearchSourceBuilder sort(String name, SortOrder order) { + if (name.equals(ScoreSortBuilder.NAME)) { + return sort(SortBuilders.scoreSort().order(order)); + } return sort(SortBuilders.fieldSort(name).order(order)); } @@ -346,32 +350,27 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ * The name of the field to sort by */ public SearchSourceBuilder sort(String name) { + if (name.equals(ScoreSortBuilder.NAME)) { + return sort(SortBuilders.scoreSort()); + } return sort(SortBuilders.fieldSort(name)); } /** * Adds a sort builder. */ - public SearchSourceBuilder sort(SortBuilder sort) { - try { + public SearchSourceBuilder sort(SortBuilder sort) { if (sorts == null) { sorts = new ArrayList<>(); } - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - sort.toXContent(builder, EMPTY_PARAMS); - builder.endObject(); - sorts.add(builder.bytes()); + sorts.add(sort); return this; - } catch (IOException e) { - throw new RuntimeException(e); - } } /** * Gets the bytes representing the sort builders for this request. */ - public List sorts() { + public List> sorts() { return sorts; } @@ -907,9 +906,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ } else if (context.parseFieldMatcher().match(currentFieldName, SUGGEST_FIELD)) { suggestBuilder = SuggestBuilder.fromXContent(context, suggesters); } else if (context.parseFieldMatcher().match(currentFieldName, SORT_FIELD)) { - sorts = new ArrayList<>(); - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); - sorts.add(xContentBuilder.bytes()); + sorts = new ArrayList<>(SortBuilder.fromXContent(context)); } else if (context.parseFieldMatcher().match(currentFieldName, EXT_FIELD)) { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); ext = xContentBuilder.bytes(); @@ -940,11 +937,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ } } } else if (context.parseFieldMatcher().match(currentFieldName, SORT_FIELD)) { - sorts = new ArrayList<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); - sorts.add(xContentBuilder.bytes()); - } + sorts = new ArrayList<>(SortBuilder.fromXContent(context)); } else if (context.parseFieldMatcher().match(currentFieldName, RESCORE_FIELD)) { rescoreBuilders = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { @@ -1057,10 +1050,10 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ if (sorts != null) { builder.startArray(SORT_FIELD.getPreferredName()); - for (BytesReference sort : sorts) { - XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(sort); - parser.nextToken(); - builder.copyCurrentStructure(parser); + for (SortBuilder sort : sorts) { + builder.startObject(); + sort.toXContent(builder, params); + builder.endObject(); } builder.endArray(); } @@ -1266,9 +1259,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ builder.size = in.readVInt(); if (in.readBoolean()) { int size = in.readVInt(); - List sorts = new ArrayList<>(); + List> sorts = new ArrayList<>(); for (int i = 0; i < size; i++) { - sorts.add(in.readBytesReference()); + sorts.add(in.readSortBuilder()); } builder.sorts = sorts; } @@ -1382,8 +1375,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ out.writeBoolean(hasSorts); if (hasSorts) { out.writeVInt(sorts.size()); - for (BytesReference sort : sorts) { - out.writeBytesReference(sort); + for (SortBuilder sort : sorts) { + out.writeSortBuilder(sort); } } boolean hasStats = stats != null; diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java index 39c432f9cc9..f9cf3f09f39 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java @@ -35,7 +35,6 @@ import org.elasticsearch.search.highlight.HighlighterParseElement; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.InternalSearchHits; import org.elasticsearch.search.internal.SearchContext; -import org.elasticsearch.search.sort.SortParseElement; import java.io.IOException; import java.util.HashMap; @@ -51,8 +50,8 @@ public class InnerHitsFetchSubPhase implements FetchSubPhase { private FetchPhase fetchPhase; @Inject - public InnerHitsFetchSubPhase(SortParseElement sortParseElement, FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement, ScriptFieldsParseElement scriptFieldsParseElement) { - parseElements = singletonMap("inner_hits", new InnerHitsParseElement(sortParseElement, sourceParseElement, highlighterParseElement, + public InnerHitsFetchSubPhase(FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement, ScriptFieldsParseElement scriptFieldsParseElement) { + parseElements = singletonMap("inner_hits", new InnerHitsParseElement(sourceParseElement, highlighterParseElement, fieldDataFieldsParseElement, scriptFieldsParseElement)); } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java index 077268ac52f..fc1b2bf399f 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java @@ -32,7 +32,6 @@ import org.elasticsearch.search.fetch.source.FetchSourceParseElement; import org.elasticsearch.search.highlight.HighlighterParseElement; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.SubSearchContext; -import org.elasticsearch.search.sort.SortParseElement; import java.util.HashMap; import java.util.Map; @@ -43,14 +42,12 @@ import static org.elasticsearch.index.query.support.InnerHitsQueryParserHelper.p */ public class InnerHitsParseElement implements SearchParseElement { - private final SortParseElement sortParseElement; private final FetchSourceParseElement sourceParseElement; private final HighlighterParseElement highlighterParseElement; private final FieldDataFieldsParseElement fieldDataFieldsParseElement; private final ScriptFieldsParseElement scriptFieldsParseElement; - public InnerHitsParseElement(SortParseElement sortParseElement, FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement, ScriptFieldsParseElement scriptFieldsParseElement) { - this.sortParseElement = sortParseElement; + public InnerHitsParseElement(FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement, ScriptFieldsParseElement scriptFieldsParseElement) { this.sourceParseElement = sourceParseElement; this.highlighterParseElement = highlighterParseElement; this.fieldDataFieldsParseElement = fieldDataFieldsParseElement; @@ -184,10 +181,10 @@ public class InnerHitsParseElement implements SearchParseElement { } else if ("inner_hits".equals(fieldName)) { childInnerHits = parseInnerHits(parser, context, searchContext); } else { - parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sortParseElement, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement); + parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement); } } else { - parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sortParseElement, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement); + parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement); } } diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 5a98744505a..f40f253e3ae 100644 --- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -58,7 +58,6 @@ import org.elasticsearch.search.profile.ProfileShardResult; import org.elasticsearch.search.profile.Profiler; import org.elasticsearch.search.rescore.RescorePhase; import org.elasticsearch.search.rescore.RescoreSearchContext; -import org.elasticsearch.search.sort.SortParseElement; import org.elasticsearch.search.sort.TrackScoresParseElement; import org.elasticsearch.search.suggest.SuggestPhase; @@ -98,7 +97,6 @@ public class QueryPhase implements SearchPhase { parseElements.put("query", new QueryParseElement()); parseElements.put("post_filter", new PostFilterParseElement()); parseElements.put("postFilter", new PostFilterParseElement()); - parseElements.put("sort", new SortParseElement()); parseElements.put("trackScores", new TrackScoresParseElement()); parseElements.put("track_scores", new TrackScoresParseElement()); parseElements.put("min_score", new MinScoreParseElement()); diff --git a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index fe946521ea4..f6c191cb74a 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -20,12 +20,10 @@ package org.elasticsearch.search.sort; import org.apache.lucene.search.SortField; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -44,7 +42,7 @@ import java.util.Objects; * A sort builder to sort based on a document field. */ public class FieldSortBuilder extends SortBuilder { - static final FieldSortBuilder PROTOTYPE = new FieldSortBuilder(""); + public static final FieldSortBuilder PROTOTYPE = new FieldSortBuilder(""); public static final String NAME = "field_sort"; public static final ParseField NESTED_PATH = new ParseField("nested_path"); public static final ParseField NESTED_FILTER = new ParseField("nested_filter"); @@ -109,19 +107,12 @@ public class FieldSortBuilder extends SortBuilder { * _first to sort missing last or first respectively. */ public FieldSortBuilder missing(Object missing) { - if (missing instanceof String) { - this.missing = BytesRefs.toBytesRef(missing); - } else { - this.missing = missing; - } + this.missing = missing; return this; } /** Returns the value used when a field is missing in a doc. */ public Object missing() { - if (missing instanceof BytesRef) { - return ((BytesRef) missing).utf8ToString(); - } return missing; } @@ -211,11 +202,7 @@ public class FieldSortBuilder extends SortBuilder { builder.startObject(fieldName); builder.field(ORDER_FIELD.getPreferredName(), order); if (missing != null) { - if (missing instanceof BytesRef) { - builder.field(MISSING.getPreferredName(), ((BytesRef) missing).utf8ToString()); - } else { - builder.field(MISSING.getPreferredName(), missing); - } + builder.field(MISSING.getPreferredName(), missing); } if (unmappedType != null) { builder.field(UNMAPPED_TYPE.getPreferredName(), unmappedType); @@ -376,7 +363,7 @@ public class FieldSortBuilder extends SortBuilder { if (context.parseFieldMatcher().match(currentFieldName, NESTED_PATH)) { nestedPath = parser.text(); } else if (context.parseFieldMatcher().match(currentFieldName, MISSING)) { - missing = parser.objectBytes(); + missing = parser.objectText(); } else if (context.parseFieldMatcher().match(currentFieldName, REVERSE)) { if (parser.booleanValue()) { order = SortOrder.DESC; diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java index df8eb1f2d7c..21217129cc4 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java @@ -74,7 +74,7 @@ public class GeoDistanceSortBuilder extends SortBuilder public static final ParseField NESTED_PATH_FIELD = new ParseField("nested_path"); public static final ParseField NESTED_FILTER_FIELD = new ParseField("nested_filter"); - static final GeoDistanceSortBuilder PROTOTYPE = new GeoDistanceSortBuilder("", -1, -1); + public static final GeoDistanceSortBuilder PROTOTYPE = new GeoDistanceSortBuilder("_na_", -1, -1); private final String fieldName; private final List points = new ArrayList<>(); diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java deleted file mode 100644 index aff0e68fc1d..00000000000 --- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.sort; - -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.join.BitSetProducer; -import org.apache.lucene.util.BitSet; -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.Version; -import org.elasticsearch.common.geo.GeoDistance; -import org.elasticsearch.common.geo.GeoDistance.FixedSourceDistance; -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.geo.GeoUtils; -import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.common.unit.DistanceUnit; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; -import org.elasticsearch.index.fielddata.NumericDoubleValues; -import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.support.NestedInnerQueryParseSupport; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * - */ -public class GeoDistanceSortParser implements SortParser { - - @Override - public String[] names() { - return new String[]{"_geo_distance", "_geoDistance"}; - } - - @Override - public SortField parse(XContentParser parser, QueryShardContext context) throws IOException { - String fieldName = null; - List geoPoints = new ArrayList<>(); - DistanceUnit unit = DistanceUnit.DEFAULT; - GeoDistance geoDistance = GeoDistance.DEFAULT; - boolean reverse = false; - MultiValueMode sortMode = null; - NestedInnerQueryParseSupport nestedHelper = null; - - final boolean indexCreatedBeforeV2_0 = context.indexVersionCreated().before(Version.V_2_0_0); - boolean coerce = GeoDistanceSortBuilder.DEFAULT_COERCE; - boolean ignoreMalformed = GeoDistanceSortBuilder.DEFAULT_IGNORE_MALFORMED; - - XContentParser.Token token; - String currentName = parser.currentName(); - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentName = parser.currentName(); - } else if (token == XContentParser.Token.START_ARRAY) { - GeoDistanceSortBuilder.parseGeoPoints(parser, geoPoints); - - fieldName = currentName; - } else if (token == XContentParser.Token.START_OBJECT) { - // the json in the format of -> field : { lat : 30, lon : 12 } - if ("nested_filter".equals(currentName) || "nestedFilter".equals(currentName)) { - if (nestedHelper == null) { - nestedHelper = new NestedInnerQueryParseSupport(parser, context); - } - nestedHelper.filter(); - } else { - fieldName = currentName; - GeoPoint point = new GeoPoint(); - GeoUtils.parseGeoPoint(parser, point); - geoPoints.add(point); - } - } else if (token.isValue()) { - if ("reverse".equals(currentName)) { - reverse = parser.booleanValue(); - } else if ("order".equals(currentName)) { - reverse = "desc".equals(parser.text()); - } else if (currentName.equals("unit")) { - unit = DistanceUnit.fromString(parser.text()); - } else if (currentName.equals("distance_type") || currentName.equals("distanceType")) { - geoDistance = GeoDistance.fromString(parser.text()); - } else if ("coerce".equals(currentName) || (indexCreatedBeforeV2_0 && "normalize".equals(currentName))) { - coerce = parser.booleanValue(); - if (coerce == true) { - ignoreMalformed = true; - } - } else if ("ignore_malformed".equals(currentName)) { - boolean ignoreMalformedFlag = parser.booleanValue(); - if (coerce == false) { - ignoreMalformed = ignoreMalformedFlag; - } - } else if ("sort_mode".equals(currentName) || "sortMode".equals(currentName) || "mode".equals(currentName)) { - sortMode = MultiValueMode.fromString(parser.text()); - } else if ("nested_path".equals(currentName) || "nestedPath".equals(currentName)) { - if (nestedHelper == null) { - nestedHelper = new NestedInnerQueryParseSupport(parser, context); - } - nestedHelper.setPath(parser.text()); - } else { - GeoPoint point = new GeoPoint(); - point.resetFromString(parser.text()); - geoPoints.add(point); - fieldName = currentName; - } - } - } - - // validation was not available prior to 2.x, so to support bwc percolation queries we only ignore_malformed on 2.x created indexes - if (!indexCreatedBeforeV2_0 && !ignoreMalformed) { - for (GeoPoint point : geoPoints) { - if (point.lat() > 90.0 || point.lat() < -90.0) { - throw new ElasticsearchParseException("illegal latitude value [{}] for [GeoDistanceSort]", point.lat()); - } - if (point.lon() > 180.0 || point.lon() < -180) { - throw new ElasticsearchParseException("illegal longitude value [{}] for [GeoDistanceSort]", point.lon()); - } - } - } - - if (coerce) { - for (GeoPoint point : geoPoints) { - GeoUtils.normalizePoint(point, coerce, coerce); - } - } - - if (sortMode == null) { - sortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN; - } - - if (sortMode == MultiValueMode.SUM) { - throw new IllegalArgumentException("sort_mode [sum] isn't supported for sorting by geo distance"); - } - - MappedFieldType fieldType = context.fieldMapper(fieldName); - if (fieldType == null) { - throw new IllegalArgumentException("failed to find mapper for [" + fieldName + "] for geo distance based sort"); - } - final MultiValueMode finalSortMode = sortMode; // final reference for use in the anonymous class - final IndexGeoPointFieldData geoIndexFieldData = context.getForField(fieldType); - final FixedSourceDistance[] distances = new FixedSourceDistance[geoPoints.size()]; - for (int i = 0; i< geoPoints.size(); i++) { - distances[i] = geoDistance.fixedSourceDistance(geoPoints.get(i).lat(), geoPoints.get(i).lon(), unit); - } - - final Nested nested; - if (nestedHelper != null && nestedHelper.getPath() != null) { - BitSetProducer rootDocumentsFilter = context.bitsetFilter(Queries.newNonNestedFilter()); - Query innerDocumentsQuery; - if (nestedHelper.filterFound()) { - // TODO: use queries instead - innerDocumentsQuery = nestedHelper.getInnerFilter(); - } else { - innerDocumentsQuery = nestedHelper.getNestedObjectMapper().nestedTypeFilter(); - } - - nested = new Nested(rootDocumentsFilter, innerDocumentsQuery); - } else { - nested = null; - } - - IndexFieldData.XFieldComparatorSource geoDistanceComparatorSource = new IndexFieldData.XFieldComparatorSource() { - - @Override - public SortField.Type reducedType() { - return SortField.Type.DOUBLE; - } - - @Override - public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - return new FieldComparator.DoubleComparator(numHits, null, null) { - @Override - protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException { - final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues(); - final SortedNumericDoubleValues distanceValues = GeoDistance.distanceValues(geoPointValues, distances); - final NumericDoubleValues selectedValues; - if (nested == null) { - selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE); - } else { - final BitSet rootDocs = nested.rootDocs(context); - final DocIdSetIterator innerDocs = nested.innerDocs(context); - selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE, rootDocs, innerDocs, context.reader().maxDoc()); - } - return selectedValues.getRawDoubleValues(); - } - }; - } - - }; - - return new SortField(fieldName, geoDistanceComparatorSource, reverse); - } - -} diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java index b35b9b94164..c0205935cf8 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java @@ -39,7 +39,7 @@ import java.util.Objects; public class ScoreSortBuilder extends SortBuilder { public static final String NAME = "_score"; - static final ScoreSortBuilder PROTOTYPE = new ScoreSortBuilder(); + public static final ScoreSortBuilder PROTOTYPE = new ScoreSortBuilder(); public static final ParseField REVERSE_FIELD = new ParseField("reverse"); public static final ParseField ORDER_FIELD = new ParseField("order"); private static final SortField SORT_SCORE = new SortField(null, SortField.Type.SCORE); diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java index b905af881b3..283c3c28390 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java @@ -67,7 +67,7 @@ import java.util.Objects; public class ScriptSortBuilder extends SortBuilder { public static final String NAME = "_script"; - static final ScriptSortBuilder PROTOTYPE = new ScriptSortBuilder(new Script("_na_"), ScriptSortType.STRING); + public static final ScriptSortBuilder PROTOTYPE = new ScriptSortBuilder(new Script("_na_"), ScriptSortType.STRING); public static final ParseField TYPE_FIELD = new ParseField("type"); public static final ParseField SCRIPT_FIELD = new ParseField("script"); public static final ParseField SORTMODE_FIELD = new ParseField("mode"); diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java deleted file mode 100644 index af8531da87c..00000000000 --- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.sort; - -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.join.BitSetProducer; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.FieldData; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.NumericDoubleValues; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; -import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.support.NestedInnerQueryParseSupport; -import org.elasticsearch.script.LeafSearchScript; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.Script.ScriptField; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptParameterParser; -import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue; -import org.elasticsearch.script.SearchScript; -import org.elasticsearch.search.MultiValueMode; -import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -/** - * - */ -public class ScriptSortParser implements SortParser { - - @Override - public String[] names() { - return new String[]{"_script"}; - } - - @Override - public SortField parse(XContentParser parser, QueryShardContext context) throws IOException { - ScriptParameterParser scriptParameterParser = new ScriptParameterParser(); - Script script = null; - ScriptSortType type = null; - Map params = null; - boolean reverse = false; - MultiValueMode sortMode = null; - NestedInnerQueryParseSupport nestedHelper = null; - - XContentParser.Token token; - String currentName = parser.currentName(); - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - if (context.parseFieldMatcher().match(currentName, ScriptField.SCRIPT)) { - script = Script.parse(parser, context.parseFieldMatcher()); - } else if ("params".equals(currentName)) { - params = parser.map(); - } else if ("nested_filter".equals(currentName) || "nestedFilter".equals(currentName)) { - if (nestedHelper == null) { - nestedHelper = new NestedInnerQueryParseSupport(parser, context); - } - nestedHelper.filter(); - } - } else if (token.isValue()) { - if ("reverse".equals(currentName)) { - reverse = parser.booleanValue(); - } else if ("order".equals(currentName)) { - reverse = "desc".equals(parser.text()); - } else if (scriptParameterParser.token(currentName, token, parser, context.parseFieldMatcher())) { - // Do Nothing (handled by ScriptParameterParser - } else if ("type".equals(currentName)) { - type = ScriptSortType.fromString(parser.text()); - } else if ("mode".equals(currentName)) { - sortMode = MultiValueMode.fromString(parser.text()); - } else if ("nested_path".equals(currentName) || "nestedPath".equals(currentName)) { - if (nestedHelper == null) { - nestedHelper = new NestedInnerQueryParseSupport(parser, context); - } - nestedHelper.setPath(parser.text()); - } - } - } - - if (script == null) { // Didn't find anything using the new API so try using the old one instead - ScriptParameterValue scriptValue = scriptParameterParser.getDefaultScriptParameterValue(); - if (scriptValue != null) { - if (params == null) { - params = new HashMap<>(); - } - script = new Script(scriptValue.script(), scriptValue.scriptType(), scriptParameterParser.lang(), params); - } - } else if (params != null) { - throw new ParsingException(parser.getTokenLocation(), "script params must be specified inside script object"); - } - - if (script == null) { - throw new ParsingException(parser.getTokenLocation(), "_script sorting requires setting the script to sort by"); - } - if (type == null) { - throw new ParsingException(parser.getTokenLocation(), "_script sorting requires setting the type of the script"); - } - final SearchScript searchScript = context.getScriptService().search( - context.lookup(), script, ScriptContext.Standard.SEARCH, Collections.emptyMap()); - - if (ScriptSortType.STRING.equals(type) && (sortMode == MultiValueMode.SUM || sortMode == MultiValueMode.AVG)) { - throw new ParsingException(parser.getTokenLocation(), "type [string] doesn't support mode [" + sortMode + "]"); - } - - if (sortMode == null) { - sortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN; - } - - final Nested nested; - if (nestedHelper != null && nestedHelper.getPath() != null) { - BitSetProducer rootDocumentsFilter = context.bitsetFilter(Queries.newNonNestedFilter()); - Query innerDocumentsFilter; - if (nestedHelper.filterFound()) { - // TODO: use queries instead - innerDocumentsFilter = nestedHelper.getInnerFilter(); - } else { - innerDocumentsFilter = nestedHelper.getNestedObjectMapper().nestedTypeFilter(); - } - nested = new Nested(rootDocumentsFilter, innerDocumentsFilter); - } else { - nested = null; - } - - final IndexFieldData.XFieldComparatorSource fieldComparatorSource; - switch (type) { - case STRING: - fieldComparatorSource = new BytesRefFieldComparatorSource(null, null, sortMode, nested) { - LeafSearchScript leafScript; - @Override - protected SortedBinaryDocValues getValues(LeafReaderContext context) throws IOException { - leafScript = searchScript.getLeafSearchScript(context); - final BinaryDocValues values = new BinaryDocValues() { - final BytesRefBuilder spare = new BytesRefBuilder(); - @Override - public BytesRef get(int docID) { - leafScript.setDocument(docID); - spare.copyChars(leafScript.run().toString()); - return spare.get(); - } - }; - return FieldData.singleton(values, null); - } - @Override - protected void setScorer(Scorer scorer) { - leafScript.setScorer(scorer); - } - }; - break; - case NUMBER: - fieldComparatorSource = new DoubleValuesComparatorSource(null, Double.MAX_VALUE, sortMode, nested) { - LeafSearchScript leafScript; - @Override - protected SortedNumericDoubleValues getValues(LeafReaderContext context) throws IOException { - leafScript = searchScript.getLeafSearchScript(context); - final NumericDoubleValues values = new NumericDoubleValues() { - @Override - public double get(int docID) { - leafScript.setDocument(docID); - return leafScript.runAsDouble(); - } - }; - return FieldData.singleton(values, null); - } - @Override - protected void setScorer(Scorer scorer) { - leafScript.setScorer(scorer); - } - }; - break; - default: - throw new ParsingException(parser.getTokenLocation(), "custom script sort type [" + type + "] not supported"); - } - - return new SortField("_script", fieldComparatorSource, reverse); - } -} diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java index c966a343751..e007ac7736e 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.sort; import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.join.BitSetProducer; import org.elasticsearch.action.support.ToXContentToBytes; @@ -34,6 +35,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.ArrayList; @@ -41,6 +43,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import static java.util.Collections.unmodifiableMap; @@ -157,6 +160,41 @@ public abstract class SortBuilder> extends ToXContentTo } } + public static void parseSort(XContentParser parser, SearchContext context) throws IOException { + QueryParseContext parseContext = context.getQueryShardContext().parseContext(); + parseContext.reset(parser); + Optional sortOptional = buildSort(SortBuilder.fromXContent(parseContext), context.getQueryShardContext()); + if (sortOptional.isPresent()) { + context.sort(sortOptional.get()); + } + } + + public static Optional buildSort(List> sortBuilders, QueryShardContext context) throws IOException { + List sortFields = new ArrayList<>(sortBuilders.size()); + for (SortBuilder builder : sortBuilders) { + sortFields.add(builder.build(context)); + } + if (!sortFields.isEmpty()) { + // optimize if we just sort on score non reversed, we don't really + // need sorting + boolean sort; + if (sortFields.size() > 1) { + sort = true; + } else { + SortField sortField = sortFields.get(0); + if (sortField.getType() == SortField.Type.SCORE && !sortField.getReverse()) { + sort = false; + } else { + sort = true; + } + } + if (sort) { + return Optional.of(new Sort(sortFields.toArray(new SortField[sortFields.size()]))); + } + } + return Optional.empty(); + } + protected static Nested resolveNested(QueryShardContext context, String nestedPath, QueryBuilder nestedFilter) throws IOException { Nested nested = null; if (nestedPath != null) { diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java b/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java deleted file mode 100644 index 1ed2a457a5f..00000000000 --- a/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.sort; - -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.join.BitSetProducer; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.QueryShardException; -import org.elasticsearch.index.query.support.NestedInnerQueryParseSupport; -import org.elasticsearch.search.MultiValueMode; -import org.elasticsearch.search.SearchParseElement; -import org.elasticsearch.search.internal.SearchContext; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static java.util.Collections.unmodifiableMap; - -/** - * - */ -public class SortParseElement implements SearchParseElement { - - private static final SortField SORT_SCORE = new SortField(null, SortField.Type.SCORE); - private static final SortField SORT_SCORE_REVERSE = new SortField(null, SortField.Type.SCORE, true); - private static final SortField SORT_DOC = new SortField(null, SortField.Type.DOC); - private static final SortField SORT_DOC_REVERSE = new SortField(null, SortField.Type.DOC, true); - - public static final ParseField UNMAPPED_TYPE = new ParseField("unmapped_type"); - - public static final String SCORE_FIELD_NAME = "_score"; - public static final String DOC_FIELD_NAME = "_doc"; - - private static final Map PARSERS; - - static { - Map parsers = new HashMap<>(); - addParser(parsers, new ScriptSortParser()); - addParser(parsers, new GeoDistanceSortParser()); - PARSERS = unmodifiableMap(parsers); - } - - private static void addParser(Map parsers, SortParser parser) { - for (String name : parser.names()) { - parsers.put(name, parser); - } - } - - @Override - public void parse(XContentParser parser, SearchContext context) throws IOException { - List sortFields = parse(parser, context.getQueryShardContext()); - if (!sortFields.isEmpty()) { - // optimize if we just sort on score non reversed, we don't really need sorting - boolean sort; - if (sortFields.size() > 1) { - sort = true; - } else { - SortField sortField = sortFields.get(0); - if (sortField.getType() == SortField.Type.SCORE && !sortField.getReverse()) { - sort = false; - } else { - sort = true; - } - } - if (sort) { - context.sort(new Sort(sortFields.toArray(new SortField[sortFields.size()]))); - } - } - } - - List parse(XContentParser parser, QueryShardContext context) throws IOException { - XContentParser.Token token = parser.currentToken(); - List sortFields = new ArrayList<>(2); - if (token == XContentParser.Token.START_ARRAY) { - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - if (token == XContentParser.Token.START_OBJECT) { - addCompoundSortField(parser, context, sortFields); - } else if (token == XContentParser.Token.VALUE_STRING) { - addSortField(context, sortFields, parser.text(), false, null, null, null, null); - } else { - throw new IllegalArgumentException("malformed sort format, within the sort array, an object, or an actual string are allowed"); - } - } - } else if (token == XContentParser.Token.VALUE_STRING) { - addSortField(context, sortFields, parser.text(), false, null, null, null, null); - } else if (token == XContentParser.Token.START_OBJECT) { - addCompoundSortField(parser, context, sortFields); - } else { - throw new IllegalArgumentException("malformed sort format, either start with array, object, or an actual string"); - } - return sortFields; - } - - private void addCompoundSortField(XContentParser parser, QueryShardContext context, List sortFields) throws IOException { - XContentParser.Token token; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - String fieldName = parser.currentName(); - boolean reverse = false; - String missing = null; - String innerJsonName = null; - String unmappedType = null; - MultiValueMode sortMode = null; - NestedInnerQueryParseSupport nestedFilterParseHelper = null; - token = parser.nextToken(); - if (token == XContentParser.Token.VALUE_STRING) { - String direction = parser.text(); - if (direction.equals("asc")) { - reverse = SCORE_FIELD_NAME.equals(fieldName); - } else if (direction.equals("desc")) { - reverse = !SCORE_FIELD_NAME.equals(fieldName); - } else { - throw new IllegalArgumentException("sort direction [" + fieldName + "] not supported"); - } - addSortField(context, sortFields, fieldName, reverse, unmappedType, missing, sortMode, nestedFilterParseHelper); - } else { - if (PARSERS.containsKey(fieldName)) { - sortFields.add(PARSERS.get(fieldName).parse(parser, context)); - } else { - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - innerJsonName = parser.currentName(); - } else if (token.isValue()) { - if ("reverse".equals(innerJsonName)) { - reverse = parser.booleanValue(); - } else if ("order".equals(innerJsonName)) { - if ("asc".equals(parser.text())) { - reverse = SCORE_FIELD_NAME.equals(fieldName); - } else if ("desc".equals(parser.text())) { - reverse = !SCORE_FIELD_NAME.equals(fieldName); - } - } else if ("missing".equals(innerJsonName)) { - missing = parser.textOrNull(); - } else if (context.parseFieldMatcher().match(innerJsonName, UNMAPPED_TYPE)) { - unmappedType = parser.textOrNull(); - } else if ("mode".equals(innerJsonName)) { - sortMode = MultiValueMode.fromString(parser.text()); - } else if ("nested_path".equals(innerJsonName) || "nestedPath".equals(innerJsonName)) { - if (nestedFilterParseHelper == null) { - nestedFilterParseHelper = new NestedInnerQueryParseSupport(parser, context); - } - nestedFilterParseHelper.setPath(parser.text()); - } else { - throw new IllegalArgumentException("sort option [" + innerJsonName + "] not supported"); - } - } else if (token == XContentParser.Token.START_OBJECT) { - if ("nested_filter".equals(innerJsonName) || "nestedFilter".equals(innerJsonName)) { - if (nestedFilterParseHelper == null) { - nestedFilterParseHelper = new NestedInnerQueryParseSupport(parser, context); - } - nestedFilterParseHelper.filter(); - } else { - throw new IllegalArgumentException("sort option [" + innerJsonName + "] not supported"); - } - } - } - addSortField(context, sortFields, fieldName, reverse, unmappedType, missing, sortMode, nestedFilterParseHelper); - } - } - } - } - } - - private void addSortField(QueryShardContext context, List sortFields, String fieldName, boolean reverse, String unmappedType, @Nullable final String missing, MultiValueMode sortMode, NestedInnerQueryParseSupport nestedHelper) throws IOException { - if (SCORE_FIELD_NAME.equals(fieldName)) { - if (reverse) { - sortFields.add(SORT_SCORE_REVERSE); - } else { - sortFields.add(SORT_SCORE); - } - } else if (DOC_FIELD_NAME.equals(fieldName)) { - if (reverse) { - sortFields.add(SORT_DOC_REVERSE); - } else { - sortFields.add(SORT_DOC); - } - } else { - MappedFieldType fieldType = context.fieldMapper(fieldName); - if (fieldType == null) { - if (unmappedType != null) { - fieldType = context.getMapperService().unmappedFieldType(unmappedType); - } else { - throw new QueryShardException(context, "No mapping found for [" + fieldName + "] in order to sort on"); - } - } - - if (!fieldType.isSortable()) { - throw new QueryShardException(context, "Sorting not supported for field[" + fieldName + "]"); - } - - // We only support AVG and SUM on number based fields - if (fieldType.isNumeric() == false && (sortMode == MultiValueMode.SUM || sortMode == MultiValueMode.AVG)) { - sortMode = null; - } - if (sortMode == null) { - sortMode = resolveDefaultSortMode(reverse); - } - - final Nested nested; - if (nestedHelper != null && nestedHelper.getPath() != null) { - BitSetProducer rootDocumentsFilter = context.bitsetFilter(Queries.newNonNestedFilter()); - Query innerDocumentsQuery; - if (nestedHelper.filterFound()) { - innerDocumentsQuery = nestedHelper.getInnerFilter(); - } else { - innerDocumentsQuery = nestedHelper.getNestedObjectMapper().nestedTypeFilter(); - } - nested = new Nested(rootDocumentsFilter, innerDocumentsQuery); - } else { - nested = null; - } - - IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.getForField(fieldType) - .comparatorSource(missing, sortMode, nested); - sortFields.add(new SortField(fieldType.name(), fieldComparatorSource, reverse)); - } - } - - private static MultiValueMode resolveDefaultSortMode(boolean reverse) { - return reverse ? MultiValueMode.MAX : MultiValueMode.MIN; - } - -} diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortParser.java b/core/src/main/java/org/elasticsearch/search/sort/SortParser.java deleted file mode 100644 index 519d9adb957..00000000000 --- a/core/src/main/java/org/elasticsearch/search/sort/SortParser.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.sort; - -import org.apache.lucene.search.SortField; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.query.QueryShardContext; - -import java.io.IOException; - -/** - * - */ -public interface SortParser { - - String[] names(); - - SortField parse(XContentParser parser, QueryShardContext context) throws IOException; -} diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index df992a06aae..74b7b17a037 100644 --- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -74,6 +74,8 @@ import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.HighlightBuilderTests; import org.elasticsearch.search.rescore.QueryRescoreBuilderTests; import org.elasticsearch.search.searchafter.SearchAfterBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; @@ -551,7 +553,7 @@ public class SearchSourceBuilderTests extends ESTestCase { SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.parseSearchSource(parser, createParseContext(parser), aggParsers, suggesters); assertEquals(1, searchSourceBuilder.sorts().size()); - assertEquals("{\"foo\":{\"order\":\"asc\"}}", searchSourceBuilder.sorts().get(0).toUtf8()); + assertEquals(new FieldSortBuilder("foo"), searchSourceBuilder.sorts().get(0)); } } @@ -567,11 +569,11 @@ public class SearchSourceBuilderTests extends ESTestCase { SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.parseSearchSource(parser, createParseContext(parser), aggParsers, suggesters); assertEquals(5, searchSourceBuilder.sorts().size()); - assertEquals("{\"post_date\":{\"order\":\"asc\"}}", searchSourceBuilder.sorts().get(0).toUtf8()); - assertEquals("\"user\"", searchSourceBuilder.sorts().get(1).toUtf8()); - assertEquals("{\"name\":\"desc\"}", searchSourceBuilder.sorts().get(2).toUtf8()); - assertEquals("{\"age\":\"desc\"}", searchSourceBuilder.sorts().get(3).toUtf8()); - assertEquals("\"_score\"", searchSourceBuilder.sorts().get(4).toUtf8()); + assertEquals(new FieldSortBuilder("post_date"), searchSourceBuilder.sorts().get(0)); + assertEquals(new FieldSortBuilder("user"), searchSourceBuilder.sorts().get(1)); + assertEquals(new FieldSortBuilder("name").order(SortOrder.DESC), searchSourceBuilder.sorts().get(2)); + assertEquals(new FieldSortBuilder("age").order(SortOrder.DESC), searchSourceBuilder.sorts().get(3)); + assertEquals(new ScoreSortBuilder(), searchSourceBuilder.sorts().get(4)); } } } diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index c3e31bf2899..e6161bc8163 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -66,7 +66,6 @@ import org.junit.BeforeClass; import java.io.IOException; import java.nio.file.Path; import java.util.Collections; -import java.util.List; import java.util.Map; import static org.hamcrest.Matchers.equalTo; @@ -78,7 +77,6 @@ public abstract class AbstractSortTestCase> extends EST private static final int NUMBER_OF_TESTBUILDERS = 20; static IndicesQueriesRegistry indicesQueriesRegistry; - private static SortParseElement parseElement = new SortParseElement(); private static ScriptService scriptService; @BeforeClass @@ -163,24 +161,12 @@ public abstract class AbstractSortTestCase> extends EST for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { T sortBuilder = createTestItem(); SortField sortField = sortBuilder.build(mockShardContext); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - if (randomBoolean()) { - builder.prettyPrint(); - } - builder.startObject(); - sortBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject(); - XContentParser parser = XContentHelper.createParser(builder.bytes()); - parser.nextToken(); - List sortFields = parseElement.parse(parser, mockShardContext); - assertEquals(1, sortFields.size()); - SortField sortFieldOldStyle = sortFields.get(0); - assertEquals(sortFieldOldStyle.getField(), sortField.getField()); - assertEquals(sortFieldOldStyle.getReverse(), sortField.getReverse()); - assertEquals(sortFieldOldStyle.getType(), sortField.getType()); + sortFieldAssertions(sortBuilder, sortField); } } + protected abstract void sortFieldAssertions(T builder, SortField sortField) throws IOException; + /** * Test serialization and deserialization of the test sort. */ diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java index 74b56353a91..2c1a4a65c7b 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java @@ -19,6 +19,8 @@ x * Licensed to Elasticsearch under one or more contributor package org.elasticsearch.search.sort; +import org.apache.lucene.search.SortField; + import java.io.IOException; public class FieldSortBuilderTests extends AbstractSortTestCase { @@ -29,7 +31,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase Date: Wed, 23 Mar 2016 11:31:45 +0100 Subject: [PATCH 10/45] SortBuilder#toXContent should render full object --- .../percolate/PercolateRequestBuilder.java | 3 +-- .../percolate/PercolateSourceBuilder.java | 18 ++++++++---------- .../tophits/TopHitsAggregatorBuilder.java | 2 -- .../search/builder/SearchSourceBuilder.java | 2 -- .../search/sort/FieldSortBuilder.java | 4 +++- .../search/sort/GeoDistanceSortBuilder.java | 2 ++ .../search/sort/ScoreSortBuilder.java | 2 ++ .../search/sort/ScriptSortBuilder.java | 2 ++ .../search/sort/AbstractSortTestCase.java | 3 --- .../search/sort/SortBuilderTests.java | 4 ---- 10 files changed, 18 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java index 83757dab089..9286601da69 100644 --- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java @@ -21,7 +21,6 @@ package org.elasticsearch.action.percolate; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; @@ -127,7 +126,7 @@ public class PercolateRequestBuilder extends ActionRequestBuilder sort) { sourceBuilder().addSort(sort); return this; } diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java b/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java index d687d48fb0c..5a5924f7883 100644 --- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java @@ -48,13 +48,13 @@ import java.util.Map; public class PercolateSourceBuilder extends ToXContentToBytes { private DocBuilder docBuilder; - private QueryBuilder queryBuilder; + private QueryBuilder queryBuilder; private Integer size; - private List sorts; + private List> sorts; private Boolean trackScores; private HighlightBuilder highlightBuilder; private List> aggregationBuilders; - private List pipelineAggregationBuilders; + private List> pipelineAggregationBuilders; /** * Sets the document to run the percolate queries against. @@ -68,7 +68,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes { * Sets a query to reduce the number of percolate queries to be evaluated and score the queries that match based * on this query. */ - public PercolateSourceBuilder setQueryBuilder(QueryBuilder queryBuilder) { + public PercolateSourceBuilder setQueryBuilder(QueryBuilder queryBuilder) { this.queryBuilder = queryBuilder; return this; } @@ -98,7 +98,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes { * * By default the matching percolator queries are returned in an undefined order. */ - public PercolateSourceBuilder addSort(SortBuilder sort) { + public PercolateSourceBuilder addSort(SortBuilder sort) { if (sorts == null) { sorts = new ArrayList<>(); } @@ -137,7 +137,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes { /** * Add an aggregation definition. */ - public PercolateSourceBuilder addAggregation(PipelineAggregatorBuilder aggregationBuilder) { + public PercolateSourceBuilder addAggregation(PipelineAggregatorBuilder aggregationBuilder) { if (pipelineAggregationBuilders == null) { pipelineAggregationBuilders = new ArrayList<>(); } @@ -160,10 +160,8 @@ public class PercolateSourceBuilder extends ToXContentToBytes { } if (sorts != null) { builder.startArray("sort"); - for (SortBuilder sort : sorts) { - builder.startObject(); + for (SortBuilder sort : sorts) { sort.toXContent(builder, params); - builder.endObject(); } builder.endArray(); } @@ -182,7 +180,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes { } } if (pipelineAggregationBuilders != null) { - for (PipelineAggregatorBuilder aggregation : pipelineAggregationBuilders) { + for (PipelineAggregatorBuilder aggregation : pipelineAggregationBuilders) { aggregation.toXContent(builder, params); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java index 279de65fcae..fd60fa93857 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java @@ -502,9 +502,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder sort : sorts) { - builder.startObject(); sort.toXContent(builder, params); - builder.endObject(); } builder.endArray(); } diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 67d25958310..eae12def5e4 100644 --- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -1051,9 +1051,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ if (sorts != null) { builder.startArray(SORT_FIELD.getPreferredName()); for (SortBuilder sort : sorts) { - builder.startObject(); sort.toXContent(builder, params); - builder.endObject(); } builder.endArray(); } diff --git a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index f6c191cb74a..59d9287d71d 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -42,7 +42,7 @@ import java.util.Objects; * A sort builder to sort based on a document field. */ public class FieldSortBuilder extends SortBuilder { - public static final FieldSortBuilder PROTOTYPE = new FieldSortBuilder(""); + public static final FieldSortBuilder PROTOTYPE = new FieldSortBuilder("_na_"); public static final String NAME = "field_sort"; public static final ParseField NESTED_PATH = new ParseField("nested_path"); public static final ParseField NESTED_FILTER = new ParseField("nested_filter"); @@ -199,6 +199,7 @@ public class FieldSortBuilder extends SortBuilder { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); builder.startObject(fieldName); builder.field(ORDER_FIELD.getPreferredName(), order); if (missing != null) { @@ -217,6 +218,7 @@ public class FieldSortBuilder extends SortBuilder { builder.field(NESTED_PATH.getPreferredName(), nestedPath); } builder.endObject(); + builder.endObject(); return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java index 21217129cc4..f3b6a1d49fa 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java @@ -300,6 +300,7 @@ public class GeoDistanceSortBuilder extends SortBuilder @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); builder.startObject(NAME); builder.startArray(fieldName); @@ -325,6 +326,7 @@ public class GeoDistanceSortBuilder extends SortBuilder builder.field(COERCE_FIELD.getPreferredName(), coerce); builder.field(IGNORE_MALFORMED_FIELD.getPreferredName(), ignoreMalformed); + builder.endObject(); builder.endObject(); return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java index c0205935cf8..57371a5cc11 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java @@ -53,9 +53,11 @@ public class ScoreSortBuilder extends SortBuilder { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); builder.startObject(NAME); builder.field(ORDER_FIELD.getPreferredName(), order); builder.endObject(); + builder.endObject(); return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java index 283c3c28390..b79eb6e2147 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java @@ -179,6 +179,7 @@ public class ScriptSortBuilder extends SortBuilder { @Override public XContentBuilder toXContent(XContentBuilder builder, Params builderParams) throws IOException { + builder.startObject(); builder.startObject(NAME); builder.field(SCRIPT_FIELD.getPreferredName(), script); builder.field(TYPE_FIELD.getPreferredName(), type); @@ -193,6 +194,7 @@ public class ScriptSortBuilder extends SortBuilder { builder.field(NESTED_FILTER_FIELD.getPreferredName(), nestedFilter, builderParams); } builder.endObject(); + builder.endObject(); return builder; } diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index e6161bc8163..935dfb178b5 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -129,10 +129,7 @@ public abstract class AbstractSortTestCase> extends EST if (randomBoolean()) { builder.prettyPrint(); } - builder.startObject(); testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject(); - XContentParser itemParser = XContentHelper.createParser(builder.bytes()); itemParser.nextToken(); diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java index 6ddbc678a12..ccb52820e3a 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java @@ -160,15 +160,11 @@ public class SortBuilderTests extends ESTestCase { xContentBuilder.endObject(); break; case 2: - xContentBuilder.startObject(); builder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); - xContentBuilder.endObject(); break; } } else { - xContentBuilder.startObject(); builder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); - xContentBuilder.endObject(); } } if (testBuilders.size() > 1) { From c2ba4448b077953f98b0d905cdc836e6916580ec Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 23 Mar 2016 14:42:10 +0100 Subject: [PATCH 11/45] Fix test bug. --- .../index/mapper/core/StringMappingUpgradeTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java index f98c74a7d25..92cee7c5407 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java @@ -126,7 +126,7 @@ public class StringMappingUpgradeTests extends ESSingleNodeTestCase { .field("type", "string") .field("index", keyword ? "not_analyzed" : "analyzed") .startObject("fielddata") - .field("forwat", format) + .field("format", format) .field("loading", loading) .startObject("filter") .startObject("frequency") From b139f4e0bf7566932746126beb06423a5b539b93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Kr=C3=A1l?= Date: Tue, 22 Mar 2016 20:58:41 +0100 Subject: [PATCH 12/45] [TEST] Move yaml test requiring yaml, add skip:yaml Clients don't ship with yaml (de)serializer by default so this test must be optionally skipped --- .../test/cat.aliases/10_basic.yaml | 25 ---------------- .../test/cat.aliases/30_yaml.yaml | 29 +++++++++++++++++++ .../test/rest/support/Features.java | 2 +- 3 files changed, 30 insertions(+), 26 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml index 423ca29a002..770c5c8b441 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml @@ -49,31 +49,6 @@ - \s+ $/ ---- -"Simple alias with yaml body through format argument": - - - do: - indices.create: - index: test - - - do: - indices.put_alias: - index: test - name: test_alias - - - do: - cat.aliases: - format: yaml - - - match: - $body: | - /^---\n - -\s+alias:\s+"test_alias"\s+ - index:\s+"test"\s+ - filter:\s+"-"\s+ - routing.index:\s+"-"\s+ - routing.search:\s+"-"\s+$/ - --- "Complex alias": diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml new file mode 100644 index 00000000000..c892891f08f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml @@ -0,0 +1,29 @@ +--- +"Simple alias with yaml body through format argument": + + - skip: + features: yaml + + - do: + indices.create: + index: test + + - do: + indices.put_alias: + index: test + name: test_alias + + - do: + cat.aliases: + format: yaml + + - match: + $body: | + /^---\n + -\s+alias:\s+"test_alias"\s+ + index:\s+"test"\s+ + filter:\s+"-"\s+ + routing.index:\s+"-"\s+ + routing.search:\s+"-"\s+$/ + + diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java index 0f51f72e8e5..66ba1528b90 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java @@ -34,7 +34,7 @@ import java.util.List; */ public final class Features { - private static final List SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers"); + private static final List SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers", "yaml"); private Features() { From 940fc3f86457f725c1f5a103c00c94a9e800323f Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 23 Mar 2016 14:44:20 +0100 Subject: [PATCH 13/45] Move dynamic template matching logic to the MatchType enum. #17281 --- .../index/mapper/object/DynamicTemplate.java | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java b/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java index 4a1ab2027b7..eaaa1ddfc69 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java @@ -41,12 +41,20 @@ public class DynamicTemplate implements ToXContent { public static enum MatchType { SIMPLE { + @Override + public boolean matches(String pattern, String value) { + return Regex.simpleMatch(pattern, value); + } @Override public String toString() { return "simple"; } }, REGEX { + @Override + public boolean matches(String pattern, String value) { + return value.matches(pattern); + } @Override public String toString() { return "regex"; @@ -61,6 +69,9 @@ public class DynamicTemplate implements ToXContent { } throw new IllegalArgumentException("No matching pattern matched on [" + value + "]"); } + + /** Whether {@code value} matches {@code regex}. */ + public abstract boolean matches(String regex, String value); } public static DynamicTemplate parse(String name, Map conf, @@ -137,23 +148,23 @@ public class DynamicTemplate implements ToXContent { } public boolean match(ContentPath path, String name, String dynamicType) { - if (pathMatch != null && !patternMatch(pathMatch, path.pathAsText(name))) { + if (pathMatch != null && !matchType.matches(pathMatch, path.pathAsText(name))) { return false; } - if (match != null && !patternMatch(match, name)) { + if (match != null && !matchType.matches(match, name)) { return false; } - if (pathUnmatch != null && patternMatch(pathUnmatch, path.pathAsText(name))) { + if (pathUnmatch != null && matchType.matches(pathUnmatch, path.pathAsText(name))) { return false; } - if (unmatch != null && patternMatch(unmatch, name)) { + if (unmatch != null && matchType.matches(unmatch, name)) { return false; } if (matchMappingType != null) { if (dynamicType == null) { return false; } - if (!patternMatch(matchMappingType, dynamicType)) { + if (!matchType.matches(matchMappingType, dynamicType)) { return false; } } @@ -186,13 +197,6 @@ public class DynamicTemplate implements ToXContent { return type; } - private boolean patternMatch(String pattern, String str) { - if (matchType == MatchType.SIMPLE) { - return Regex.simpleMatch(pattern, str); - } - return str.matches(pattern); - } - public Map mappingForName(String name, String dynamicType) { return processMap(mapping, name, dynamicType); } From e319985d935fc1c2af426040fae297260b8df343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 23 Mar 2016 15:27:53 +0100 Subject: [PATCH 14/45] Adding test for parsing sort on single fields as list --- .../org/elasticsearch/search/sort/SortBuilderTests.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java index ccb52820e3a..b8100b58155 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java @@ -82,6 +82,13 @@ public class SortBuilderTests extends ESTestCase { sortBuilder = result.get(0); assertEquals(new FieldSortBuilder("field1"), sortBuilder); + // one element array, see https://github.com/elastic/elasticsearch/issues/17257 + json = "{ \"sort\" : [\"field1\"] }"; + result = parseSort(json); + assertEquals(1, result.size()); + sortBuilder = result.get(0); + assertEquals(new FieldSortBuilder("field1"), sortBuilder); + json = "{ \"sort\" : { \"_doc\" : \"" + order + "\" }}"; result = parseSort(json); assertEquals(1, result.size()); From f91a046b6fbe30a2589df4045479c1ec4f2471c7 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 23 Mar 2016 09:03:29 -0700 Subject: [PATCH 15/45] Cli: Reject positional argument for bin/elasticsearch This exits with a usage error when bin/elasticsearch recieves any positional args. closes #17287 --- .../elasticsearch/bootstrap/Elasticsearch.java | 3 +++ .../bootstrap/ElasticsearchCliTests.java | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index 0cc952907c0..bb1f6cc87d5 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -76,6 +76,9 @@ class Elasticsearch extends Command { @Override protected void execute(Terminal terminal, OptionSet options) throws Exception { + if (options.nonOptionArguments().isEmpty() == false) { + throw new UserError(ExitCodes.USAGE, "Positional arguments not allowed, found " + options.nonOptionArguments()); + } if (options.has(versionOption)) { if (options.has(daemonizeOption) || options.has(pidfileOption)) { throw new UserError(ExitCodes.USAGE, "Elasticsearch version option is mutually exclusive with any other option"); diff --git a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java index 51274af9a01..96ef26eacc6 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java @@ -76,6 +76,23 @@ public class ElasticsearchCliTests extends ESTestCase { runTest(expectedStatus, false, outputConsumer, (foreground, pidFile, esSettings) -> {}, args); } + public void testPositionalArgs() throws Exception { + runTest( + ExitCodes.USAGE, + false, + output -> assertThat(output, containsString("Positional args not allowed, found [foo]")), + (foreground, pidFile, esSettings) -> {}, + "foo" + ); + runTest( + ExitCodes.USAGE, + false, + output -> assertThat(output, containsString("Positional args not allowed, found [foo, bar]")), + (foreground, pidFile, esSettings) -> {}, + "foo", "bar" + ); + } + public void testThatPidFileCanBeConfigured() throws Exception { runPidFileTest(ExitCodes.USAGE, false, output -> assertThat(output, containsString("Option p/pidfile requires an argument")), "-p"); runPidFileTest(ExitCodes.OK, true, output -> {}, "-p", "/tmp/pid"); From 4b1ae331f04e1cf47871b81f6821d7ac1d4fb811 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 23 Mar 2016 17:32:51 +0100 Subject: [PATCH 16/45] Update after review --- .../java/org/elasticsearch/repositories/s3/S3Repository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 6742233cf78..a09d57ebc93 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -266,7 +266,7 @@ public class S3Repository extends BlobStoreRepository { this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING); // We make sure that chunkSize is bigger or equal than/to bufferSize - if (this.chunkSize.getMb() < bufferSize.getMb()) { + if (this.chunkSize.getBytes() < bufferSize.getBytes()) { throw new RepositoryException(name.name(), Repository.CHUNK_SIZE_SETTING.getKey() + " (" + this.chunkSize + ") can't be lower than " + Repository.BUFFER_SIZE_SETTING.getKey() + " (" + bufferSize + ")."); } From 3f73ef9aa47a8c2c9a027b37424532e8fd0302ec Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 23 Mar 2016 12:35:01 -0400 Subject: [PATCH 17/45] Protect max size virtual memory check from Windows --- .../main/java/org/elasticsearch/bootstrap/BootstrapCheck.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java index 5f4e980d68e..576ce720f35 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java @@ -123,7 +123,9 @@ final class BootstrapCheck { if (Constants.LINUX) { checks.add(new MaxNumberOfThreadsCheck()); } - checks.add(new MaxSizeVirtualMemoryCheck()); + if (Constants.LINUX || Constants.MAC_OS_X) { + checks.add(new MaxSizeVirtualMemoryCheck()); + } return Collections.unmodifiableList(checks); } From 0654b84d2cb07cfdb8919180a7aaf23d98c23f39 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 23 Mar 2016 13:23:44 -0400 Subject: [PATCH 18/45] Useful error message for null property placeholder This commit adds the key to the error message when encountering a missing property placeholder. --- .../common/property/PropertyPlaceholder.java | 4 +- .../common/settings/Settings.java | 2 +- .../property/PropertyPlaceholderTests.java | 49 ++++++++++++------- 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java b/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java index a210f3ae6d5..c4339329190 100644 --- a/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java +++ b/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java @@ -76,8 +76,8 @@ public class PropertyPlaceholder { * @param placeholderResolver the PlaceholderResolver to use for replacement. * @return the supplied value with placeholders replaced inline. */ - public String replacePlaceholders(String value, PlaceholderResolver placeholderResolver) { - Objects.requireNonNull(value, "Argument 'value' must not be null."); + public String replacePlaceholders(String key, String value, PlaceholderResolver placeholderResolver) { + Objects.requireNonNull(value, "value can not be null for [" + key + "]"); return parseStringValue(value, placeholderResolver, new HashSet()); } diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java index a6784e561d2..ce79bf92d20 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -1221,7 +1221,7 @@ public final class Settings implements ToXContent { } }; for (Map.Entry entry : new HashMap<>(map).entrySet()) { - String value = propertyPlaceholder.replacePlaceholders(entry.getValue(), placeholderResolver); + String value = propertyPlaceholder.replacePlaceholders(entry.getKey(), entry.getValue(), placeholderResolver); // if the values exists and has length, we should maintain it in the map // otherwise, the replace process resolved into removing it if (Strings.hasLength(value)) { diff --git a/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java b/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java index 4d8fbc3add5..459c192cdb7 100644 --- a/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java +++ b/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.test.ESTestCase; import java.util.LinkedHashMap; import java.util.Map; +import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.is; public class PropertyPlaceholderTests extends ESTestCase { @@ -33,10 +34,10 @@ public class PropertyPlaceholderTests extends ESTestCase { map.put("foo1", "bar1"); map.put("foo2", "bar2"); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("bar1", propertyPlaceholder.replacePlaceholders("{foo1}", placeholderResolver)); - assertEquals("a bar1b", propertyPlaceholder.replacePlaceholders("a {foo1}b", placeholderResolver)); - assertEquals("bar1bar2", propertyPlaceholder.replacePlaceholders("{foo1}{foo2}", placeholderResolver)); - assertEquals("a bar1 b bar2 c", propertyPlaceholder.replacePlaceholders("a {foo1} b {foo2} c", placeholderResolver)); + assertEquals("bar1", propertyPlaceholder.replacePlaceholders("key", "{foo1}", placeholderResolver)); + assertEquals("a bar1b", propertyPlaceholder.replacePlaceholders("key", "a {foo1}b", placeholderResolver)); + assertEquals("bar1bar2", propertyPlaceholder.replacePlaceholders("key", "{foo1}{foo2}", placeholderResolver)); + assertEquals("a bar1 b bar2 c", propertyPlaceholder.replacePlaceholders("key", "a {foo1} b {foo2} c", placeholderResolver)); } public void testVariousPrefixSuffix() { @@ -47,24 +48,24 @@ public class PropertyPlaceholderTests extends ESTestCase { Map map = new LinkedHashMap<>(); map.put("foo", "bar"); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("bar", ppEqualsPrefix.replacePlaceholders("{foo}", placeholderResolver)); - assertEquals("bar", ppLongerPrefix.replacePlaceholders("${foo}", placeholderResolver)); - assertEquals("bar", ppShorterPrefix.replacePlaceholders("{foo}}", placeholderResolver)); + assertEquals("bar", ppEqualsPrefix.replacePlaceholders("key", "{foo}", placeholderResolver)); + assertEquals("bar", ppLongerPrefix.replacePlaceholders("key", "${foo}", placeholderResolver)); + assertEquals("bar", ppShorterPrefix.replacePlaceholders("key", "{foo}}", placeholderResolver)); } public void testDefaultValue() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); Map map = new LinkedHashMap<>(); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("bar", propertyPlaceholder.replacePlaceholders("${foo:bar}", placeholderResolver)); - assertEquals("", propertyPlaceholder.replacePlaceholders("${foo:}", placeholderResolver)); + assertEquals("bar", propertyPlaceholder.replacePlaceholders("key", "${foo:bar}", placeholderResolver)); + assertEquals("", propertyPlaceholder.replacePlaceholders("key", "${foo:}", placeholderResolver)); } public void testIgnoredUnresolvedPlaceholder() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", true); Map map = new LinkedHashMap<>(); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("${foo}", propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver)); + assertEquals("${foo}", propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver)); } public void testNotIgnoredUnresolvedPlaceholder() { @@ -72,7 +73,7 @@ public class PropertyPlaceholderTests extends ESTestCase { Map map = new LinkedHashMap<>(); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); try { - propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver); + propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver); fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), is("Could not resolve placeholder 'foo'")); @@ -83,7 +84,7 @@ public class PropertyPlaceholderTests extends ESTestCase { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); Map map = new LinkedHashMap<>(); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, true); - assertEquals("bar", propertyPlaceholder.replacePlaceholders("bar${foo}", placeholderResolver)); + assertEquals("bar", propertyPlaceholder.replacePlaceholders("key", "bar${foo}", placeholderResolver)); } public void testRecursive() { @@ -93,8 +94,8 @@ public class PropertyPlaceholderTests extends ESTestCase { map.put("foo1", "${foo2}"); map.put("foo2", "bar"); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("bar", propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver)); - assertEquals("abarb", propertyPlaceholder.replacePlaceholders("a${foo}b", placeholderResolver)); + assertEquals("bar", propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver)); + assertEquals("abarb", propertyPlaceholder.replacePlaceholders("key", "a${foo}b", placeholderResolver)); } public void testNestedLongerPrefix() { @@ -105,7 +106,7 @@ public class PropertyPlaceholderTests extends ESTestCase { map.put("foo2", "bar"); map.put("barbar", "baz"); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("baz", propertyPlaceholder.replacePlaceholders("${bar${foo}}", placeholderResolver)); + assertEquals("baz", propertyPlaceholder.replacePlaceholders("key", "${bar${foo}}", placeholderResolver)); } public void testNestedSameLengthPrefixSuffix() { @@ -116,7 +117,7 @@ public class PropertyPlaceholderTests extends ESTestCase { map.put("foo2", "bar"); map.put("barbar", "baz"); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("baz", propertyPlaceholder.replacePlaceholders("{bar{foo}}", placeholderResolver)); + assertEquals("baz", propertyPlaceholder.replacePlaceholders("key", "{bar{foo}}", placeholderResolver)); } public void testNestedShorterPrefix() { @@ -127,7 +128,7 @@ public class PropertyPlaceholderTests extends ESTestCase { map.put("foo2", "bar"); map.put("barbar", "baz"); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("baz", propertyPlaceholder.replacePlaceholders("{bar{foo}}}}", placeholderResolver)); + assertEquals("baz", propertyPlaceholder.replacePlaceholders("key", "{bar{foo}}}}", placeholderResolver)); } public void testCircularReference() { @@ -137,7 +138,7 @@ public class PropertyPlaceholderTests extends ESTestCase { map.put("bar", "${foo}"); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); try { - propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver); + propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver); fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), is("Circular placeholder reference 'foo' in property definitions")); @@ -148,7 +149,17 @@ public class PropertyPlaceholderTests extends ESTestCase { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); Map map = new LinkedHashMap<>(); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, false); - assertEquals("bar${foo}", propertyPlaceholder.replacePlaceholders("bar${foo}", placeholderResolver)); + assertEquals("bar${foo}", propertyPlaceholder.replacePlaceholders("key", "bar${foo}", placeholderResolver)); + } + + public void testNullValue() { + final PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); + final Map map = new LinkedHashMap<>(); + final PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, false); + final String key = randomAsciiOfLength(10); + NullPointerException e = + expectThrows(NullPointerException.class, () -> propertyPlaceholder.replacePlaceholders(key, null, placeholderResolver)); + assertThat(e, hasToString("java.lang.NullPointerException: value can not be null for [" + key + "]")); } private class SimplePlaceholderResolver implements PropertyPlaceholder.PlaceholderResolver { From 0fe6c9f3b47f2380e607640b6126c17d146eb5a5 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 23 Mar 2016 11:52:25 -0700 Subject: [PATCH 19/45] Add elasticsearch cli test for options plus illegal args --- .../bootstrap/ElasticsearchCliTests.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java index 96ef26eacc6..632646146fe 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java @@ -80,17 +80,24 @@ public class ElasticsearchCliTests extends ESTestCase { runTest( ExitCodes.USAGE, false, - output -> assertThat(output, containsString("Positional args not allowed, found [foo]")), + output -> assertThat(output, containsString("Positional arguments not allowed, found [foo]")), (foreground, pidFile, esSettings) -> {}, "foo" ); runTest( ExitCodes.USAGE, false, - output -> assertThat(output, containsString("Positional args not allowed, found [foo, bar]")), + output -> assertThat(output, containsString("Positional arguments not allowed, found [foo, bar]")), (foreground, pidFile, esSettings) -> {}, "foo", "bar" ); + runTest( + ExitCodes.USAGE, + false, + output -> assertThat(output, containsString("Positional arguments not allowed, found [foo]")), + (foreground, pidFile, esSettings) -> {}, + "-E", "something", "foo", "-E", "somethingelse" + ); } public void testThatPidFileCanBeConfigured() throws Exception { From 0647638a99bb5842c82a6c99d6380eab65f1652f Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 23 Mar 2016 14:56:49 -0400 Subject: [PATCH 20/45] Guard against null key for property placeholder --- .../elasticsearch/common/property/PropertyPlaceholder.java | 1 + .../common/property/PropertyPlaceholderTests.java | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java b/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java index c4339329190..70e6807cb92 100644 --- a/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java +++ b/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java @@ -77,6 +77,7 @@ public class PropertyPlaceholder { * @return the supplied value with placeholders replaced inline. */ public String replacePlaceholders(String key, String value, PlaceholderResolver placeholderResolver) { + Objects.requireNonNull(key); Objects.requireNonNull(value, "value can not be null for [" + key + "]"); return parseStringValue(value, placeholderResolver, new HashSet()); } diff --git a/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java b/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java index 459c192cdb7..405ac566686 100644 --- a/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java +++ b/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java @@ -152,6 +152,13 @@ public class PropertyPlaceholderTests extends ESTestCase { assertEquals("bar${foo}", propertyPlaceholder.replacePlaceholders("key", "bar${foo}", placeholderResolver)); } + public void testNullKey() { + final PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); + final Map map = new LinkedHashMap<>(); + final PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, false); + expectThrows(NullPointerException.class, () -> propertyPlaceholder.replacePlaceholders(null, "value", placeholderResolver)); + } + public void testNullValue() { final PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); final Map map = new LinkedHashMap<>(); From 033167de5bbf7ac905b918c77fd7884ab519085f Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 23 Mar 2016 15:55:10 -0400 Subject: [PATCH 21/45] Wait for yellow indices when running upgrade test This commit makes the Vagrant upgrade test wait for yellow indices before attempting to get documents from the upgraded Elasticsearch node. --- .../resources/packaging/scripts/80_upgrade.bats | 3 ++- .../packaging/scripts/packaging_test_utils.bash | 13 ++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats b/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats index 65d212e56d8..48bf2aca4e3 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats +++ b/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats @@ -83,7 +83,8 @@ setup() { } @test "[UPGRADE] start version under test" { - start_elasticsearch_service yellow + start_elasticsearch_service yellow library + wait_for_elasticsearch_status yellow library2 } @test "[UPGRADE] check elasticsearch version is version under test" { diff --git a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash index cb18363d60a..e2a072273bf 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash +++ b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash @@ -270,10 +270,11 @@ clean_before_test() { # $1 - expected status - defaults to green start_elasticsearch_service() { local desiredStatus=${1:-green} + local index=$2 run_elasticsearch_service 0 - wait_for_elasticsearch_status $desiredStatus + wait_for_elasticsearch_status $desiredStatus $index if [ -r "/tmp/elasticsearch/elasticsearch.pid" ]; then pid=$(cat /tmp/elasticsearch/elasticsearch.pid) @@ -382,6 +383,7 @@ stop_elasticsearch_service() { # $1 - expected status - defaults to green wait_for_elasticsearch_status() { local desiredStatus=${1:-green} + local index=$2 echo "Making sure elasticsearch is up..." wget -O - --retry-connrefused --waitretry=1 --timeout=60 --tries 60 http://localhost:9200 || { @@ -395,8 +397,13 @@ wait_for_elasticsearch_status() { false } - echo "Tring to connect to elasticsearch and wait for expected status..." - curl -sS "http://localhost:9200/_cluster/health?wait_for_status=$desiredStatus&timeout=60s&pretty" + if [ -z "index" ]; then + echo "Tring to connect to elasticsearch and wait for expected status $desiredStatus..." + curl -sS "http://localhost:9200/_cluster/health?wait_for_status=$desiredStatus&timeout=60s&pretty" + else + echo "Trying to connect to elasticsearch and wait for expected status $desiredStatus for index $index" + curl -sS "http://localhost:9200/_cluster/$index/health?wait_for_status=$desiredStatus&timeout=60s&pretty" + fi if [ $? -eq 0 ]; then echo "Connected" else From 5ed2bb5f189768e950e86e3fc57fa56471f3a380 Mon Sep 17 00:00:00 2001 From: Areek Zillur Date: Fri, 18 Mar 2016 13:56:22 -0400 Subject: [PATCH 22/45] [REST] use search transport for suggest rest endpoint --- .../action/suggest/RestSuggestAction.java | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java b/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java index 53d9e668de1..1bbc662929c 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java @@ -19,14 +19,15 @@ package org.elasticsearch.rest.action.suggest; -import org.elasticsearch.action.suggest.SuggestRequest; -import org.elasticsearch.action.suggest.SuggestResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; @@ -41,6 +42,7 @@ import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestBuilderListener; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.search.suggest.Suggesters; @@ -73,28 +75,29 @@ public class RestSuggestAction extends BaseRestHandler { @Override public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException { - SuggestRequest suggestRequest = new SuggestRequest(Strings.splitStringByCommaToArray(request.param("index"))); - suggestRequest.indicesOptions(IndicesOptions.fromRequest(request, suggestRequest.indicesOptions())); + final SearchRequest searchRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index")), new SearchSourceBuilder()); + searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions())); if (RestActions.hasBodyContent(request)) { final BytesReference sourceBytes = RestActions.getRestContent(request); try (XContentParser parser = XContentFactory.xContent(sourceBytes).createParser(sourceBytes)) { final QueryParseContext context = new QueryParseContext(queryRegistry); context.reset(parser); context.parseFieldMatcher(parseFieldMatcher); - suggestRequest.suggest(SuggestBuilder.fromXContent(context, suggesters)); + searchRequest.source().suggest(SuggestBuilder.fromXContent(context, suggesters)); } } else { throw new IllegalArgumentException("no content or source provided to execute suggestion"); } - suggestRequest.routing(request.param("routing")); - suggestRequest.preference(request.param("preference")); - - client.suggest(suggestRequest, new RestBuilderListener(channel) { + searchRequest.routing(request.param("routing")); + searchRequest.preference(request.param("preference")); + client.search(searchRequest, new RestBuilderListener(channel) { @Override - public RestResponse buildResponse(SuggestResponse response, XContentBuilder builder) throws Exception { - RestStatus restStatus = RestStatus.status(response.getSuccessfulShards(), response.getTotalShards(), response.getShardFailures()); + public RestResponse buildResponse(SearchResponse response, XContentBuilder builder) throws Exception { + RestStatus restStatus = RestStatus.status(response.getSuccessfulShards(), + response.getTotalShards(), response.getShardFailures()); builder.startObject(); - buildBroadcastShardsHeader(builder, request, response); + buildBroadcastShardsHeader(builder, request, response.getTotalShards(), + response.getSuccessfulShards(), response.getFailedShards(), response.getShardFailures()); Suggest suggest = response.getSuggest(); if (suggest != null) { suggest.toInnerXContent(builder, request); From b0437c3f22e8cf540b30dd5007fd31aea19f1575 Mon Sep 17 00:00:00 2001 From: Areek Zillur Date: Fri, 18 Mar 2016 14:02:34 -0400 Subject: [PATCH 23/45] Optimize search execution for suggest-only requests We skip context preprocessing and only execute suggest phase for suggest-only requests --- .../elasticsearch/search/builder/SearchSourceBuilder.java | 8 ++++++++ .../search/internal/DefaultSearchContext.java | 3 +++ .../org/elasticsearch/search/internal/SearchContext.java | 8 ++++++++ .../java/org/elasticsearch/search/query/QueryPhase.java | 6 ++++++ 4 files changed, 25 insertions(+) diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index eae12def5e4..0093363d244 100644 --- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -729,6 +729,14 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ return ext; } + /** + * @return true if the source only has suggest + */ + public boolean hasOnlySuggest() { + return suggestBuilder != null + && queryBuilder == null && aggregations == null; + } + /** * Rewrites this search source builder into its primitive form. e.g. by * rewriting the QueryBuilder. If the builder did not change the identity diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java index 71a289331f8..f3fe48f6682 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java @@ -191,6 +191,9 @@ public class DefaultSearchContext extends SearchContext { */ @Override public void preProcess() { + if (hasOnlySuggest() ) { + return; + } if (scrollContext == null) { long from = from() == -1 ? 0 : from(); long size = size() == -1 ? 10 : size(); diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 1881109c9b6..65d41a85227 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -353,6 +353,14 @@ public abstract class SearchContext implements Releasable { } } + /** + * @return true if the request contains only suggest + */ + public final boolean hasOnlySuggest() { + return request().source() != null + && request().source().hasOnlySuggest(); + } + /** * Looks up the given field, but does not restrict to fields in the types set on this context. */ diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java index f40f253e3ae..62210655a00 100644 --- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -116,6 +116,12 @@ public class QueryPhase implements SearchPhase { @Override public void execute(SearchContext searchContext) throws QueryPhaseExecutionException { + if (searchContext.hasOnlySuggest()) { + suggestPhase.execute(searchContext); + // TODO: fix this once we can fetch docs for suggestions + searchContext.queryResult().topDocs(new TopDocs(0, Lucene.EMPTY_SCORE_DOCS, 0)); + return; + } // Pre-process aggregations as late as possible. In the case of a DFS_Q_T_F // request, preProcess is called on the DFS phase phase, this is why we pre-process them // here to make sure it happens during the QUERY phase From 0eb2032189ebadbde399e5ef5a86ecff7b9b0b4c Mon Sep 17 00:00:00 2001 From: Areek Zillur Date: Fri, 18 Mar 2016 14:41:27 -0400 Subject: [PATCH 24/45] Disable DFS search types and request caching for suggest-only requests --- .../elasticsearch/action/search/SearchRequest.java | 7 +++++++ .../action/search/TransportSearchAction.java | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 9d3c200ed98..8a3099aa84c 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -295,6 +295,13 @@ public class SearchRequest extends ActionRequest implements Indic return this.requestCache; } + /** + * @return true if the request only has suggest + */ + public boolean hasOnlySuggest() { + return source != null && source.hasOnlySuggest(); + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 8fe99c1e590..33b3cd6085d 100644 --- a/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -37,7 +37,10 @@ import org.elasticsearch.transport.TransportService; import java.util.Map; import java.util.Set; +import static org.elasticsearch.action.search.SearchType.DFS_QUERY_AND_FETCH; +import static org.elasticsearch.action.search.SearchType.DFS_QUERY_THEN_FETCH; import static org.elasticsearch.action.search.SearchType.QUERY_AND_FETCH; +import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH; /** * @@ -68,9 +71,18 @@ public class TransportSearchAction extends HandledTransportAction> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, searchRequest.routing(), searchRequest.indices()); int shardCount = clusterService.operationRouting().searchShardsCount(clusterState, concreteIndices, routingMap); + boolean hasOnlySuggest = searchRequest.hasOnlySuggest(); if (shardCount == 1) { // if we only have one group, then we always want Q_A_F, no need for DFS, and no need to do THEN since we hit one shard searchRequest.searchType(QUERY_AND_FETCH); + } else if (hasOnlySuggest && + (searchRequest.searchType() == DFS_QUERY_AND_FETCH || searchRequest.searchType() == DFS_QUERY_THEN_FETCH)) { + // convert to Q_T_F if we have only suggest + searchRequest.searchType(QUERY_THEN_FETCH); + } + if (hasOnlySuggest && (searchRequest.requestCache() == null || searchRequest.requestCache())) { + // disable request cache if we have only suggest + searchRequest.requestCache(false); } } catch (IndexNotFoundException | IndexClosedException e) { // ignore these failures, we will notify the search response if its really the case from the actual action From ed49ec437fc54e68b50909dffd7a136cabcbe239 Mon Sep 17 00:00:00 2001 From: Areek Zillur Date: Fri, 18 Mar 2016 14:45:25 -0400 Subject: [PATCH 25/45] remove suggest transport action --- .../elasticsearch/action/ActionModule.java | 3 - .../action/suggest/ShardSuggestRequest.java | 60 ------- .../action/suggest/ShardSuggestResponse.java | 61 ------- .../action/suggest/SuggestAction.java | 46 ----- .../action/suggest/SuggestRequest.java | 154 ----------------- .../action/suggest/SuggestRequestBuilder.java | 85 --------- .../action/suggest/SuggestResponse.java | 82 --------- .../suggest/TransportSuggestAction.java | 152 ---------------- .../action/suggest/package-info.java | 23 --- .../java/org/elasticsearch/client/Client.java | 26 --- .../org/elasticsearch/client/Requests.java | 12 -- .../client/support/AbstractClient.java | 19 -- .../search/suggest/SuggestPhase.java | 11 +- .../suggest/CompletionSuggestSearchIT.java | 163 ++++++++++-------- .../ContextCompletionSuggestSearchIT.java | 13 +- .../messy/tests/IndicesRequestTests.java | 14 -- .../messy/tests/SuggestSearchTests.java | 41 ++--- 17 files changed, 109 insertions(+), 856 deletions(-) delete mode 100644 core/src/main/java/org/elasticsearch/action/suggest/ShardSuggestRequest.java delete mode 100644 core/src/main/java/org/elasticsearch/action/suggest/ShardSuggestResponse.java delete mode 100644 core/src/main/java/org/elasticsearch/action/suggest/SuggestAction.java delete mode 100644 core/src/main/java/org/elasticsearch/action/suggest/SuggestRequest.java delete mode 100644 core/src/main/java/org/elasticsearch/action/suggest/SuggestRequestBuilder.java delete mode 100644 core/src/main/java/org/elasticsearch/action/suggest/SuggestResponse.java delete mode 100644 core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java delete mode 100644 core/src/main/java/org/elasticsearch/action/suggest/package-info.java diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java index be9387f9a8a..2b33a669428 100644 --- a/core/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java @@ -173,8 +173,6 @@ import org.elasticsearch.action.search.TransportClearScrollAction; import org.elasticsearch.action.search.TransportMultiSearchAction; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.search.TransportSearchScrollAction; -import org.elasticsearch.action.suggest.SuggestAction; -import org.elasticsearch.action.suggest.TransportSuggestAction; import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.AutoCreateIndex; @@ -320,7 +318,6 @@ public class ActionModule extends AbstractModule { registerAction(MultiTermVectorsAction.INSTANCE, TransportMultiTermVectorsAction.class, TransportShardMultiTermsVectorAction.class); registerAction(DeleteAction.INSTANCE, TransportDeleteAction.class); - registerAction(SuggestAction.INSTANCE, TransportSuggestAction.class); registerAction(UpdateAction.INSTANCE, TransportUpdateAction.class); registerAction(MultiGetAction.INSTANCE, TransportMultiGetAction.class, TransportShardMultiGetAction.class); diff --git a/core/src/main/java/org/elasticsearch/action/suggest/ShardSuggestRequest.java b/core/src/main/java/org/elasticsearch/action/suggest/ShardSuggestRequest.java deleted file mode 100644 index ad15bd27440..00000000000 --- a/core/src/main/java/org/elasticsearch/action/suggest/ShardSuggestRequest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.suggest; - -import org.elasticsearch.action.support.broadcast.BroadcastShardRequest; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.suggest.SuggestBuilder; - -import java.io.IOException; - -/** - * Internal suggest request executed directly against a specific index shard. - */ -public final class ShardSuggestRequest extends BroadcastShardRequest { - - private SuggestBuilder suggest; - - public ShardSuggestRequest() { - } - - ShardSuggestRequest(ShardId shardId, SuggestRequest request) { - super(shardId, request); - this.suggest = request.suggest(); - } - - public SuggestBuilder suggest() { - return suggest; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - suggest = SuggestBuilder.PROTOTYPE.readFrom(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - suggest.writeTo(out); - } -} diff --git a/core/src/main/java/org/elasticsearch/action/suggest/ShardSuggestResponse.java b/core/src/main/java/org/elasticsearch/action/suggest/ShardSuggestResponse.java deleted file mode 100644 index bca29800bd1..00000000000 --- a/core/src/main/java/org/elasticsearch/action/suggest/ShardSuggestResponse.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.suggest; - -import org.elasticsearch.action.support.broadcast.BroadcastShardResponse; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.suggest.Suggest; - -import java.io.IOException; - -/** - * Internal suggest response of a shard suggest request executed directly against a specific shard. - */ -class ShardSuggestResponse extends BroadcastShardResponse { - - private final Suggest suggest; - - ShardSuggestResponse() { - this.suggest = new Suggest(); - } - - ShardSuggestResponse(ShardId shardId, Suggest suggest) { - super(shardId); - this.suggest = suggest; - } - - public Suggest getSuggest() { - return this.suggest; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - suggest.readFrom(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - suggest.writeTo(out); - } -} diff --git a/core/src/main/java/org/elasticsearch/action/suggest/SuggestAction.java b/core/src/main/java/org/elasticsearch/action/suggest/SuggestAction.java deleted file mode 100644 index 88d2a92f331..00000000000 --- a/core/src/main/java/org/elasticsearch/action/suggest/SuggestAction.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.suggest; - -import org.elasticsearch.action.Action; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.search.suggest.Suggest; - -/** - */ -public class SuggestAction extends Action { - - public static final SuggestAction INSTANCE = new SuggestAction(); - public static final String NAME = "indices:data/read/suggest"; - - private SuggestAction() { - super(NAME); - } - - @Override - public SuggestResponse newResponse() { - return new SuggestResponse(new Suggest()); - } - - @Override - public SuggestRequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new SuggestRequestBuilder(client, this); - } -} diff --git a/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequest.java b/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequest.java deleted file mode 100644 index 1398dd1dcf1..00000000000 --- a/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequest.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.suggest; - -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.support.broadcast.BroadcastRequest; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.search.suggest.SuggestBuilder; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Objects; - -/** - * A request to get suggestions for corrections of phrases. Best created with - * {@link org.elasticsearch.client.Requests#suggestRequest(String...)}. - *

- * The request requires the suggest query source to be set using - * {@link #suggest(org.elasticsearch.search.suggest.SuggestBuilder)} - * - * @see SuggestResponse - * @see org.elasticsearch.client.Client#suggest(SuggestRequest) - * @see org.elasticsearch.client.Requests#suggestRequest(String...) - * @see org.elasticsearch.search.suggest.SuggestBuilders - */ -public final class SuggestRequest extends BroadcastRequest { - - @Nullable - private String routing; - - @Nullable - private String preference; - - private SuggestBuilder suggest; - - public SuggestRequest() { - } - - /** - * Constructs a new suggest request against the provided indices. No indices provided means it will - * run against all indices. - */ - public SuggestRequest(String... indices) { - super(indices); - } - - @Override - public ActionRequestValidationException validate() { - ActionRequestValidationException validationException = super.validate(); - return validationException; - } - - /** - * The suggestion query to get correction suggestions for - */ - public SuggestBuilder suggest() { - return suggest; - } - - /** - * set a new source for the suggest query - */ - public SuggestRequest suggest(SuggestBuilder suggest) { - Objects.requireNonNull(suggest, "suggest must not be null"); - this.suggest = suggest; - return this; - } - - /** - * A comma separated list of routing values to control the shards the search will be executed on. - */ - public String routing() { - return this.routing; - } - - /** - * A comma separated list of routing values to control the shards the search will be executed on. - */ - public SuggestRequest routing(String routing) { - this.routing = routing; - return this; - } - - /** - * The routing values to control the shards that the search will be executed on. - */ - public SuggestRequest routing(String... routings) { - this.routing = Strings.arrayToCommaDelimitedString(routings); - return this; - } - - public SuggestRequest preference(String preference) { - this.preference = preference; - return this; - } - - public String preference() { - return this.preference; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - routing = in.readOptionalString(); - preference = in.readOptionalString(); - suggest = SuggestBuilder.PROTOTYPE.readFrom(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - Objects.requireNonNull(suggest, "suggest must not be null"); - super.writeTo(out); - out.writeOptionalString(routing); - out.writeOptionalString(preference); - suggest.writeTo(out); - } - - @Override - public String toString() { - Objects.requireNonNull(suggest, "suggest must not be null"); - String sSource = "_na_"; - try { - XContentBuilder builder = JsonXContent.contentBuilder(); - builder = suggest.toXContent(builder, ToXContent.EMPTY_PARAMS); - sSource = builder.string(); - } catch (Exception e) { - // ignore - } - return "[" + Arrays.toString(indices) + "]" + ", suggest[" + sSource + "]"; - } -} diff --git a/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequestBuilder.java deleted file mode 100644 index b64745bf400..00000000000 --- a/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequestBuilder.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.suggest; - -import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.search.suggest.SuggestBuilder; -import org.elasticsearch.search.suggest.SuggestionBuilder; - -/** - * A suggest action request builder. - */ -public class SuggestRequestBuilder extends BroadcastOperationRequestBuilder { - - final SuggestBuilder suggest = new SuggestBuilder(); - - public SuggestRequestBuilder(ElasticsearchClient client, SuggestAction action) { - super(client, action, new SuggestRequest()); - } - - /** - * Add a definition for suggestions to the request - * @param name the name for the suggestion that will also be used in the response - * @param suggestion the suggestion configuration - */ - public SuggestRequestBuilder addSuggestion(String name, SuggestionBuilder suggestion) { - suggest.addSuggestion(name, suggestion); - return this; - } - - /** - * A comma separated list of routing values to control the shards the search will be executed on. - */ - public SuggestRequestBuilder setRouting(String routing) { - request.routing(routing); - return this; - } - - public SuggestRequestBuilder setSuggestText(String globalText) { - this.suggest.setGlobalText(globalText); - return this; - } - - /** - * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, - * _shards:x,y to operate on shards x & y, or a custom value, which guarantees that the same order - * will be used across different requests. - */ - public SuggestRequestBuilder setPreference(String preference) { - request.preference(preference); - return this; - } - - /** - * The routing values to control the shards that the search will be executed on. - */ - public SuggestRequestBuilder setRouting(String... routing) { - request.routing(routing); - return this; - } - - @Override - protected SuggestRequest beforeExecute(SuggestRequest request) { - request.suggest(suggest); - return request; - } -} diff --git a/core/src/main/java/org/elasticsearch/action/suggest/SuggestResponse.java b/core/src/main/java/org/elasticsearch/action/suggest/SuggestResponse.java deleted file mode 100644 index 445e804b5b5..00000000000 --- a/core/src/main/java/org/elasticsearch/action/suggest/SuggestResponse.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.suggest; - -import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.support.broadcast.BroadcastResponse; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.search.suggest.Suggest; - -import java.io.IOException; -import java.util.List; - -import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS; - -/** - * The response of the suggest action. - */ -public final class SuggestResponse extends BroadcastResponse { - - private final Suggest suggest; - - SuggestResponse(Suggest suggest) { - this.suggest = suggest; - } - - SuggestResponse(Suggest suggest, int totalShards, int successfulShards, int failedShards, List shardFailures) { - super(totalShards, successfulShards, failedShards, shardFailures); - this.suggest = suggest; - } - - /** - * The Suggestions of the phrase. - */ - public Suggest getSuggest() { - return suggest; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - this.suggest.readFrom(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - this.suggest.writeTo(out); - } - - @Override - public String toString() { - try { - XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); - builder.startObject(); - suggest.toXContent(builder, EMPTY_PARAMS); - builder.endObject(); - return builder.string(); - } catch (IOException e) { - return "{ \"error\" : \"" + e.getMessage() + "\"}"; - } - } -} diff --git a/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java b/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java deleted file mode 100644 index 95bf111ed71..00000000000 --- a/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.suggest; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.DefaultShardOperationFailedException; -import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; -import org.elasticsearch.action.support.broadcast.TransportBroadcastAction; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.GroupShardsIterator; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.suggest.stats.ShardSuggestMetric; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.search.suggest.Suggest; -import org.elasticsearch.search.suggest.SuggestBuilder; -import org.elasticsearch.search.suggest.SuggestPhase; -import org.elasticsearch.search.suggest.SuggestionSearchContext; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicReferenceArray; - -/** - * Defines the transport of a suggestion request across the cluster - */ -public class TransportSuggestAction - extends TransportBroadcastAction { - - private final IndicesService indicesService; - private final SuggestPhase suggestPhase; - - @Inject - public TransportSuggestAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, - TransportService transportService, IndicesService indicesService, SuggestPhase suggestPhase, - ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { - super(settings, SuggestAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, - SuggestRequest::new, ShardSuggestRequest::new, ThreadPool.Names.SUGGEST); - this.indicesService = indicesService; - this.suggestPhase = suggestPhase; - } - - @Override - protected ShardSuggestRequest newShardRequest(int numShards, ShardRouting shard, SuggestRequest request) { - return new ShardSuggestRequest(shard.shardId(), request); - } - - @Override - protected ShardSuggestResponse newShardResponse() { - return new ShardSuggestResponse(); - } - - @Override - protected GroupShardsIterator shards(ClusterState clusterState, SuggestRequest request, String[] concreteIndices) { - Map> routingMap = - indexNameExpressionResolver.resolveSearchRouting(clusterState, request.routing(), request.indices()); - return clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, request.preference()); - } - - @Override - protected ClusterBlockException checkGlobalBlock(ClusterState state, SuggestRequest request) { - return state.blocks().globalBlockedException(ClusterBlockLevel.READ); - } - - @Override - protected ClusterBlockException checkRequestBlock(ClusterState state, SuggestRequest countRequest, String[] concreteIndices) { - return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, concreteIndices); - } - - @Override - protected SuggestResponse newResponse(SuggestRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) { - int successfulShards = 0; - int failedShards = 0; - - final Map> groupedSuggestions = new HashMap<>(); - - List shardFailures = null; - for (int i = 0; i < shardsResponses.length(); i++) { - Object shardResponse = shardsResponses.get(i); - if (shardResponse == null) { - // simply ignore non active shards - } else if (shardResponse instanceof BroadcastShardOperationFailedException) { - failedShards++; - if (shardFailures == null) { - shardFailures = new ArrayList<>(); - } - shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse)); - } else { - Suggest suggest = ((ShardSuggestResponse) shardResponse).getSuggest(); - Suggest.group(groupedSuggestions, suggest); - successfulShards++; - } - } - - return new SuggestResponse(new Suggest(Suggest.reduce(groupedSuggestions)), shardsResponses.length(), - successfulShards, failedShards, shardFailures); - } - - @Override - protected ShardSuggestResponse shardOperation(ShardSuggestRequest request) { - IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); - IndexShard indexShard = indexService.getShard(request.shardId().id()); - ShardSuggestMetric suggestMetric = indexShard.getSuggestMetric(); - suggestMetric.preSuggest(); - long startTime = System.nanoTime(); - try (Engine.Searcher searcher = indexShard.acquireSearcher("suggest")) { - SuggestBuilder suggest = request.suggest(); - if (suggest != null) { - final SuggestionSearchContext context = suggest.build(indexService.newQueryShardContext()); - final Suggest result = suggestPhase.execute(context, searcher.searcher()); - return new ShardSuggestResponse(request.shardId(), result); - } - return new ShardSuggestResponse(request.shardId(), new Suggest()); - } catch (Throwable ex) { - throw new ElasticsearchException("failed to execute suggest", ex); - } finally { - suggestMetric.postSuggest(System.nanoTime() - startTime); - } - } -} diff --git a/core/src/main/java/org/elasticsearch/action/suggest/package-info.java b/core/src/main/java/org/elasticsearch/action/suggest/package-info.java deleted file mode 100644 index a2c0f48ea51..00000000000 --- a/core/src/main/java/org/elasticsearch/action/suggest/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * Suggest action. - */ -package org.elasticsearch.action.suggest; \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/client/Client.java b/core/src/main/java/org/elasticsearch/client/Client.java index e5d8d4f55b7..1c7676bc91c 100644 --- a/core/src/main/java/org/elasticsearch/client/Client.java +++ b/core/src/main/java/org/elasticsearch/client/Client.java @@ -68,9 +68,6 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.search.SearchScrollRequestBuilder; -import org.elasticsearch.action.suggest.SuggestRequest; -import org.elasticsearch.action.suggest.SuggestRequestBuilder; -import org.elasticsearch.action.suggest.SuggestResponse; import org.elasticsearch.action.termvectors.MultiTermVectorsRequest; import org.elasticsearch.action.termvectors.MultiTermVectorsRequestBuilder; import org.elasticsearch.action.termvectors.MultiTermVectorsResponse; @@ -367,29 +364,6 @@ public interface Client extends ElasticsearchClient, Releasable { */ MultiGetRequestBuilder prepareMultiGet(); - /** - * Suggestion matching a specific phrase. - * - * @param request The suggest request - * @return The result future - * @see Requests#suggestRequest(String...) - */ - ActionFuture suggest(SuggestRequest request); - - /** - * Suggestions matching a specific phrase. - * - * @param request The suggest request - * @param listener A listener to be notified of the result - * @see Requests#suggestRequest(String...) - */ - void suggest(SuggestRequest request, ActionListener listener); - - /** - * Suggestions matching a specific phrase. - */ - SuggestRequestBuilder prepareSuggest(String... indices); - /** * Search across one or more indices and one or more types with a query. * diff --git a/core/src/main/java/org/elasticsearch/client/Requests.java b/core/src/main/java/org/elasticsearch/client/Requests.java index 53c6abd971c..276bd9d9062 100644 --- a/core/src/main/java/org/elasticsearch/client/Requests.java +++ b/core/src/main/java/org/elasticsearch/client/Requests.java @@ -60,9 +60,7 @@ import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchScrollRequest; -import org.elasticsearch.action.suggest.SuggestRequest; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.search.suggest.SuggestBuilder; /** * A handy one stop shop for creating requests (make sure to import static this class). @@ -126,16 +124,6 @@ public class Requests { return new GetRequest(index); } - /** - * Creates a suggest request for getting suggestions from provided indices. - * The suggest query has to be set using {@link org.elasticsearch.action.suggest.SuggestRequest#suggest(SuggestBuilder)}. - * @param indices The indices to suggest from. Use null or _all to execute against all indices - * @see org.elasticsearch.client.Client#suggest(org.elasticsearch.action.suggest.SuggestRequest) - */ - public static SuggestRequest suggestRequest(String... indices) { - return new SuggestRequest(indices); - } - /** * Creates a search request against one or more indices. Note, the search source must be set either using the * actual JSON search source, or the {@link org.elasticsearch.search.builder.SearchSourceBuilder}. diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java index f729d5287df..0044890ee35 100644 --- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -314,10 +314,6 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollAction; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.search.SearchScrollRequestBuilder; -import org.elasticsearch.action.suggest.SuggestAction; -import org.elasticsearch.action.suggest.SuggestRequest; -import org.elasticsearch.action.suggest.SuggestRequestBuilder; -import org.elasticsearch.action.suggest.SuggestResponse; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.ThreadedActionListener; import org.elasticsearch.action.termvectors.MultiTermVectorsAction; @@ -660,21 +656,6 @@ public abstract class AbstractClient extends AbstractComponent implements Client return new MultiSearchRequestBuilder(this, MultiSearchAction.INSTANCE); } - @Override - public ActionFuture suggest(final SuggestRequest request) { - return execute(SuggestAction.INSTANCE, request); - } - - @Override - public void suggest(final SuggestRequest request, final ActionListener listener) { - execute(SuggestAction.INSTANCE, request, listener); - } - - @Override - public SuggestRequestBuilder prepareSuggest(String... indices) { - return new SuggestRequestBuilder(this, SuggestAction.INSTANCE).setIndices(indices); - } - @Override public ActionFuture termVectors(final TermVectorsRequest request) { return execute(TermVectorsAction.INSTANCE, request); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java index 8831d5d093a..520322587bb 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.search.suggest; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.util.CharsRefBuilder; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.component.AbstractComponent; @@ -65,10 +64,6 @@ public class SuggestPhase extends AbstractComponent implements SearchPhase { if (suggest == null) { return; } - context.queryResult().suggest(execute(suggest, context.searcher())); - } - - public Suggest execute(SuggestionSearchContext suggest, IndexSearcher searcher) { try { CharsRefBuilder spare = new CharsRefBuilder(); final List>> suggestions = new ArrayList<>(suggest.suggestions().size()); @@ -76,14 +71,14 @@ public class SuggestPhase extends AbstractComponent implements SearchPhase { for (Map.Entry entry : suggest.suggestions().entrySet()) { SuggestionSearchContext.SuggestionContext suggestion = entry.getValue(); Suggester suggester = suggestion.getSuggester(); - Suggestion> result = suggester.execute(entry.getKey(), suggestion, searcher, spare); + Suggestion> result = + suggester.execute(entry.getKey(), suggestion, context.searcher(), spare); if (result != null) { assert entry.getKey().equals(result.name); suggestions.add(result); } } - - return new Suggest(suggestions); + context.queryResult().suggest(new Suggest(suggestions)); } catch (IOException e) { throw new ElasticsearchException("I/O exception during suggest phase", e); } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java index ef0416df90d..e1f1dd02b15 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java @@ -32,7 +32,8 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; -import org.elasticsearch.action.suggest.SuggestResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Requests; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; @@ -41,6 +42,7 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.percolator.PercolatorFieldMapper; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.suggest.completion.CompletionStats; import org.elasticsearch.search.suggest.completion.CompletionSuggestion; @@ -198,9 +200,10 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg"). size(numDocs).payload(Collections.singletonList("count")); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", prefix).execute().actionGet(); - assertNoFailures(suggestResponse); - CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo"); + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix)) + .execute().actionGet(); + assertNoFailures(searchResponse); + CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo"); CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0); assertThat(options.getOptions().size(), equalTo(numDocs)); for (CompletionSuggestion.Entry.Option option : options) { @@ -219,9 +222,10 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { indexRandom(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg") .payload(Collections.singletonList("test_field")); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", prefix).execute().actionGet(); - assertNoFailures(suggestResponse); - CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo"); + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix)) + .execute().actionGet(); + assertNoFailures(searchResponse); + CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo"); CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0); assertThat(options.getOptions().size(), equalTo(2)); for (CompletionSuggestion.Entry.Option option : options.getOptions()) { @@ -257,9 +261,10 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg") .payload(Arrays.asList("title", "count")); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", prefix).execute().actionGet(); - assertNoFailures(suggestResponse); - CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo"); + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix)) + .execute().actionGet(); + assertNoFailures(searchResponse); + CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo"); List options = completionSuggestion.getEntries().get(0).getOptions(); assertThat(options.size(), equalTo(2)); assertThat(options.get(0).getText().toString(), equalTo("suggestion")); @@ -308,9 +313,10 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg") .size(suggestionSize).payload(payloadFields); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", prefix).execute().actionGet(); - assertNoFailures(suggestResponse); - CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo"); + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix)) + .execute().actionGet(); + assertNoFailures(searchResponse); + CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo"); CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0); assertThat(options.getOptions().size(), equalTo(suggestionSize)); int id = numDocs; @@ -406,12 +412,12 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("testSuggestions", - new CompletionSuggestionBuilder(FIELD).text("test").size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("testSuggestions", new CompletionSuggestionBuilder(FIELD).text("test").size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, "testSuggestions", "testing"); - Suggest.Suggestion.Entry.Option option = suggestResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0); + assertSuggestions(searchResponse, "testSuggestions", "testing"); + Suggest.Suggestion.Entry.Option option = searchResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0); assertThat(option, is(instanceOf(CompletionSuggestion.Entry.Option.class))); CompletionSuggestion.Entry.Option prefixOption = (CompletionSuggestion.Entry.Option) option; @@ -607,16 +613,16 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { .get(); assertThat(putMappingResponse.isAcknowledged(), is(true)); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("suggs", - SuggestBuilders.completionSuggestion(FIELD + ".suggest").text("f").size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("suggs", SuggestBuilders.completionSuggestion(FIELD + ".suggest").text("f").size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, "suggs"); + assertSuggestions(searchResponse, "suggs"); client().prepareIndex(INDEX, TYPE, "1").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get(); ensureGreen(INDEX); - SuggestResponse afterReindexingResponse = client().prepareSuggest(INDEX).addSuggestion("suggs", - SuggestBuilders.completionSuggestion(FIELD + ".suggest").text("f").size(10) + SearchResponse afterReindexingResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("suggs", SuggestBuilders.completionSuggestion(FIELD + ".suggest").text("f").size(10)) ).execute().actionGet(); assertSuggestions(afterReindexingResponse, "suggs", "Foo Fighters"); } @@ -632,15 +638,15 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nirv").size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("Nirv").size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + assertSuggestions(searchResponse, false, "foo", "Nirvana"); - suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nirw", Fuzziness.ONE).size(10) + searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("Nirw", Fuzziness.ONE).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + assertSuggestions(searchResponse, false, "foo", "Nirvana"); } public void testThatFuzzySuggesterSupportsEditDistances() throws Exception { @@ -655,16 +661,16 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); // edit distance 1 - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Norw", Fuzziness.ONE).size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("Norw", Fuzziness.ONE).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo"); + assertSuggestions(searchResponse, false, "foo"); // edit distance 2 - suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Norw", Fuzziness.TWO).size(10) + searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("Norw", Fuzziness.TWO).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + assertSuggestions(searchResponse, false, "foo", "Nirvana"); } public void testThatFuzzySuggesterSupportsTranspositions() throws Exception { @@ -678,15 +684,16 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nriv", FuzzyOptions.builder().setTranspositions(false).build()).size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", + SuggestBuilders.completionSuggestion(FIELD).prefix("Nriv", FuzzyOptions.builder().setTranspositions(false).build()).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo"); + assertSuggestions(searchResponse, false, "foo"); - suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nriv", Fuzziness.ONE).size(10) + searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("Nriv", Fuzziness.ONE).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + assertSuggestions(searchResponse, false, "foo", "Nirvana"); } public void testThatFuzzySuggesterSupportsMinPrefixLength() throws Exception { @@ -700,15 +707,17 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nriva", FuzzyOptions.builder().setFuzzyMinLength(6).build()).size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", + SuggestBuilders.completionSuggestion(FIELD).prefix("Nriva", FuzzyOptions.builder().setFuzzyMinLength(6).build()).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo"); + assertSuggestions(searchResponse, false, "foo"); - suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nrivan", FuzzyOptions.builder().setFuzzyMinLength(6).build()).size(10) + searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", + SuggestBuilders.completionSuggestion(FIELD).prefix("Nrivan", FuzzyOptions.builder().setFuzzyMinLength(6).build()).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + assertSuggestions(searchResponse, false, "foo", "Nirvana"); } public void testThatFuzzySuggesterSupportsNonPrefixLength() throws Exception { @@ -722,15 +731,17 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nirw", FuzzyOptions.builder().setFuzzyPrefixLength(4).build()).size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", + SuggestBuilders.completionSuggestion(FIELD).prefix("Nirw", FuzzyOptions.builder().setFuzzyPrefixLength(4).build()).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo"); + assertSuggestions(searchResponse, false, "foo"); - suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nirvo", FuzzyOptions.builder().setFuzzyPrefixLength(4).build()).size(10) + searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", + SuggestBuilders.completionSuggestion(FIELD).prefix("Nirvo", FuzzyOptions.builder().setFuzzyPrefixLength(4).build()).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + assertSuggestions(searchResponse, false, "foo", "Nirvana"); } public void testThatFuzzySuggesterIsUnicodeAware() throws Exception { @@ -748,18 +759,18 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder completionSuggestionBuilder = SuggestBuilders.completionSuggestion(FIELD).prefix("öööи", FuzzyOptions.builder().setUnicodeAware(true).build()).size(10); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", completionSuggestionBuilder).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "ööööö"); + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", completionSuggestionBuilder)).execute().actionGet(); + assertSuggestions(searchResponse, false, "foo", "ööööö"); // removing unicode awareness leads to no result completionSuggestionBuilder = SuggestBuilders.completionSuggestion(FIELD).prefix("öööи", FuzzyOptions.builder().setUnicodeAware(false).build()).size(10); - suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", completionSuggestionBuilder).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo"); + searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", completionSuggestionBuilder)).execute().actionGet(); + assertSuggestions(searchResponse, false, "foo"); // increasing edit distance instead of unicode awareness works again, as this is only a single character completionSuggestionBuilder = SuggestBuilders.completionSuggestion(FIELD).prefix("öööи", FuzzyOptions.builder().setUnicodeAware(false).setFuzziness(Fuzziness.TWO).build()).size(10); - suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", completionSuggestionBuilder).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "ööööö"); + searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", completionSuggestionBuilder)).execute().actionGet(); + assertSuggestions(searchResponse, false, "foo", "ööööö"); } public void testThatStatsAreWorking() throws Exception { @@ -787,8 +798,8 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); ensureGreen(); // load the fst index into ram - client().prepareSuggest(INDEX).addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("f")).get(); - client().prepareSuggest(INDEX).addSuggestion("foo", SuggestBuilders.completionSuggestion(otherField).prefix("f")).get(); + client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("f"))).get(); + client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(otherField).prefix("f"))).get(); // Get all stats IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(INDEX).setIndices(INDEX).setCompletion(true).get(); @@ -884,13 +895,14 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { } } - public void assertSuggestions(String suggestionName, SuggestionBuilder suggestBuilder, String... suggestions) { - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionName, suggestBuilder - ).execute().actionGet(); - assertSuggestions(suggestResponse, suggestionName, suggestions); - + final SearchRequest searchRequest = Requests.searchRequest(INDEX); + searchRequest.source(new SearchSourceBuilder()); + searchRequest.source().suggest(); + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion(suggestionName, suggestBuilder)).execute().actionGet(); + assertSuggestions(searchResponse, suggestionName, suggestions); } + public void assertSuggestions(String suggestion, String... suggestions) { String suggestionName = RandomStrings.randomAsciiOfLength(random(), 10); CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion(FIELD).text(suggestion).size(10); @@ -899,28 +911,29 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { public void assertSuggestionsNotInOrder(String suggestString, String... suggestions) { String suggestionName = RandomStrings.randomAsciiOfLength(random(), 10); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionName, - SuggestBuilders.completionSuggestion(FIELD).text(suggestString).size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion(suggestionName, + SuggestBuilders.completionSuggestion(FIELD).text(suggestString).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, suggestionName, suggestions); + assertSuggestions(searchResponse, false, suggestionName, suggestions); } - static void assertSuggestions(SuggestResponse suggestResponse, String name, String... suggestions) { - assertSuggestions(suggestResponse, true, name, suggestions); + static void assertSuggestions(SearchResponse searchResponse, String name, String... suggestions) { + assertSuggestions(searchResponse, true, name, suggestions); } - private static void assertSuggestions(SuggestResponse suggestResponse, boolean suggestionOrderStrict, String name, String... suggestions) { - assertAllSuccessful(suggestResponse); + private static void assertSuggestions(SearchResponse searchResponse, boolean suggestionOrderStrict, String name, String... suggestions) { + assertAllSuccessful(searchResponse); List suggestionNames = new ArrayList<>(); - for (Suggest.Suggestion> suggestion : iterableAsArrayList(suggestResponse.getSuggest())) { + for (Suggest.Suggestion> suggestion : iterableAsArrayList(searchResponse.getSuggest())) { suggestionNames.add(suggestion.getName()); } String expectFieldInResponseMsg = String.format(Locale.ROOT, "Expected suggestion named %s in response, got %s", name, suggestionNames); - assertThat(expectFieldInResponseMsg, suggestResponse.getSuggest().getSuggestion(name), is(notNullValue())); + assertThat(expectFieldInResponseMsg, searchResponse.getSuggest().getSuggestion(name), is(notNullValue())); - Suggest.Suggestion> suggestion = suggestResponse.getSuggest().getSuggestion(name); + Suggest.Suggestion> suggestion = searchResponse.getSuggest().getSuggestion(name); List suggestionList = getNames(suggestion.getEntries().get(0)); List options = suggestion.getEntries().get(0).getOptions(); diff --git a/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java index 58458c9d244..ee45a9d0371 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java @@ -23,7 +23,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomStrings; import org.apache.lucene.spatial.util.GeoHashUtils; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.suggest.SuggestResponse; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; @@ -624,16 +624,17 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase { String suggestionName = randomAsciiOfLength(10); CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(FIELD).text("h").size(10) .contexts(Collections.singletonMap("st", Collections.singletonList(GeoQueryContext.builder().setGeoPoint(new GeoPoint(52.52, 13.4)).build()))); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionName, context).get(); + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion(suggestionName, context)).get(); - assertEquals(suggestResponse.getSuggest().size(), 1); - assertEquals("Hotel Amsterdam in Berlin", suggestResponse.getSuggest().getSuggestion(suggestionName).iterator().next().getOptions().iterator().next().getText().string()); + assertEquals(searchResponse.getSuggest().size(), 1); + assertEquals("Hotel Amsterdam in Berlin", searchResponse.getSuggest().getSuggestion(suggestionName).iterator().next().getOptions().iterator().next().getText().string()); } public void assertSuggestions(String suggestionName, SuggestionBuilder suggestBuilder, String... suggestions) { - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionName, suggestBuilder + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion(suggestionName, suggestBuilder) ).execute().actionGet(); - CompletionSuggestSearchIT.assertSuggestions(suggestResponse, suggestionName, suggestions); + CompletionSuggestSearchIT.assertSuggestions(searchResponse, suggestionName, suggestions); } private void createIndexAndMapping(CompletionMappingBuilder completionMappingBuilder) throws IOException { diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java index 29bd7f47498..c8b30a5e2de 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java @@ -70,8 +70,6 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.action.suggest.SuggestAction; -import org.elasticsearch.action.suggest.SuggestRequest; import org.elasticsearch.action.termvectors.MultiTermVectorsAction; import org.elasticsearch.action.termvectors.MultiTermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsAction; @@ -88,7 +86,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.action.SearchTransportService; -import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -440,17 +437,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(indicesStatsRequest, indicesStats); } - public void testSuggest() { - String suggestAction = SuggestAction.NAME + "[s]"; - interceptTransportActions(suggestAction); - - SuggestRequest suggestRequest = new SuggestRequest(randomIndicesOrAliases()).suggest(new SuggestBuilder()); - internalCluster().clientNodeClient().suggest(suggestRequest).actionGet(); - - clearInterceptedActions(); - assertSameIndices(suggestRequest, suggestAction); - } - public void testValidateQuery() { String validateQueryShardAction = ValidateQueryAction.NAME + "[s]"; interceptTransportActions(validateQueryShardAction); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java index 94f60f8802c..79cac3181ea 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java @@ -60,8 +60,6 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; -import org.elasticsearch.action.suggest.SuggestRequestBuilder; -import org.elasticsearch.action.suggest.SuggestResponse; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -1269,34 +1267,17 @@ public class SuggestSearchTests extends ESIntegTestCase { } protected Suggest searchSuggest(String suggestText, int expectShardsFailed, Map> suggestions) { - if (randomBoolean()) { - SearchRequestBuilder builder = client().prepareSearch().setSize(0); - SuggestBuilder suggestBuilder = new SuggestBuilder(); - if (suggestText != null) { - suggestBuilder.setGlobalText(suggestText); - } - for (Entry> suggestion : suggestions.entrySet()) { - suggestBuilder.addSuggestion(suggestion.getKey(), suggestion.getValue()); - } - builder.suggest(suggestBuilder); - SearchResponse actionGet = builder.execute().actionGet(); - assertThat(Arrays.toString(actionGet.getShardFailures()), actionGet.getFailedShards(), equalTo(expectShardsFailed)); - return actionGet.getSuggest(); - } else { - SuggestRequestBuilder builder = client().prepareSuggest(); - if (suggestText != null) { - builder.setSuggestText(suggestText); - } - for (Entry> suggestion : suggestions.entrySet()) { - builder.addSuggestion(suggestion.getKey(), suggestion.getValue()); - } - - SuggestResponse actionGet = builder.execute().actionGet(); - assertThat(Arrays.toString(actionGet.getShardFailures()), actionGet.getFailedShards(), equalTo(expectShardsFailed)); - if (expectShardsFailed > 0) { - throw new SearchPhaseExecutionException("suggest", "Suggest execution failed", new ShardSearchFailure[0]); - } - return actionGet.getSuggest(); + SearchRequestBuilder builder = client().prepareSearch().setSize(0); + SuggestBuilder suggestBuilder = new SuggestBuilder(); + if (suggestText != null) { + suggestBuilder.setGlobalText(suggestText); } + for (Entry> suggestion : suggestions.entrySet()) { + suggestBuilder.addSuggestion(suggestion.getKey(), suggestion.getValue()); + } + builder.suggest(suggestBuilder); + SearchResponse actionGet = builder.execute().actionGet(); + assertThat(Arrays.toString(actionGet.getShardFailures()), actionGet.getFailedShards(), equalTo(expectShardsFailed)); + return actionGet.getSuggest(); } } From 91dd9b330136028f34dadda552a468a0009d87e9 Mon Sep 17 00:00:00 2001 From: Areek Zillur Date: Fri, 18 Mar 2016 14:45:49 -0400 Subject: [PATCH 26/45] Merge suggest stats into search stats --- .../admin/indices/stats/CommonStats.java | 26 +--- .../admin/indices/stats/CommonStatsFlags.java | 2 +- .../stats/IndicesStatsRequestBuilder.java | 5 - .../index/search/stats/SearchStats.java | 51 ++++++- .../index/search/stats/ShardSearchStats.java | 80 +++++------ .../elasticsearch/index/shard/IndexShard.java | 11 -- .../suggest/stats/ShardSuggestMetric.java | 56 -------- .../index/suggest/stats/SuggestStats.java | 124 ------------------ .../indices/NodeIndicesStats.java | 6 - .../indices/stats/RestIndicesStatsAction.java | 3 +- .../rest/action/cat/RestIndicesAction.java | 12 +- .../rest/action/cat/RestNodesAction.java | 8 +- .../index/suggest/stats/SuggestStatsIT.java | 44 ++++--- .../indices/IndicesOptionsIntegrationIT.java | 21 +-- .../indices/stats/IndexStatsIT.java | 16 ++- .../search/stats/SearchStatsUnitTests.java | 9 +- .../test/indices.stats/11_metric.yaml | 5 - 17 files changed, 145 insertions(+), 334 deletions(-) delete mode 100644 core/src/main/java/org/elasticsearch/index/suggest/stats/ShardSuggestMetric.java delete mode 100644 core/src/main/java/org/elasticsearch/index/suggest/stats/SuggestStats.java diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java index b6ee76b16ed..b130e6b378f 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java @@ -42,7 +42,6 @@ import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.store.StoreStats; -import org.elasticsearch.index.suggest.stats.SuggestStats; import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.index.warmer.WarmerStats; import org.elasticsearch.indices.IndicesQueryCache; @@ -109,7 +108,7 @@ public class CommonStats implements Streamable, ToXContent { translog = new TranslogStats(); break; case Suggest: - suggest = new SuggestStats(); + // skip break; case RequestCache: requestCache = new RequestCacheStats(); @@ -177,7 +176,7 @@ public class CommonStats implements Streamable, ToXContent { translog = indexShard.translogStats(); break; case Suggest: - suggest = indexShard.suggestStats(); + // skip break; case RequestCache: requestCache = indexShard.requestCache().stats(); @@ -236,9 +235,6 @@ public class CommonStats implements Streamable, ToXContent { @Nullable public TranslogStats translog; - @Nullable - public SuggestStats suggest; - @Nullable public RequestCacheStats requestCache; @@ -367,14 +363,6 @@ public class CommonStats implements Streamable, ToXContent { } else { translog.add(stats.getTranslog()); } - if (suggest == null) { - if (stats.getSuggest() != null) { - suggest = new SuggestStats(); - suggest.add(stats.getSuggest()); - } - } else { - suggest.add(stats.getSuggest()); - } if (requestCache == null) { if (stats.getRequestCache() != null) { requestCache = new RequestCacheStats(); @@ -468,11 +456,6 @@ public class CommonStats implements Streamable, ToXContent { return translog; } - @Nullable - public SuggestStats getSuggest() { - return suggest; - } - @Nullable public RequestCacheStats getRequestCache() { return requestCache; @@ -555,7 +538,6 @@ public class CommonStats implements Streamable, ToXContent { segments = SegmentsStats.readSegmentsStats(in); } translog = in.readOptionalStreamable(TranslogStats::new); - suggest = in.readOptionalStreamable(SuggestStats::new); requestCache = in.readOptionalStreamable(RequestCacheStats::new); recoveryStats = in.readOptionalStreamable(RecoveryStats::new); } @@ -647,7 +629,6 @@ public class CommonStats implements Streamable, ToXContent { segments.writeTo(out); } out.writeOptionalStreamable(translog); - out.writeOptionalStreamable(suggest); out.writeOptionalStreamable(requestCache); out.writeOptionalStreamable(recoveryStats); } @@ -700,9 +681,6 @@ public class CommonStats implements Streamable, ToXContent { if (translog != null) { translog.toXContent(builder, params); } - if (suggest != null) { - suggest.toXContent(builder, params); - } if (requestCache != null) { requestCache.toXContent(builder, params); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java index c67c35a4108..f00f22da536 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java @@ -244,7 +244,7 @@ public class CommonStatsFlags implements Streamable, Cloneable { Completion("completion"), Segments("segments"), Translog("translog"), - Suggest("suggest"), + Suggest("suggest"), // unused RequestCache("request_cache"), Recovery("recovery"); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java index 0ae21a3ac09..cad919cbd18 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java @@ -152,11 +152,6 @@ public class IndicesStatsRequestBuilder extends BroadcastOperationRequestBuilder return this; } - public IndicesStatsRequestBuilder setSuggest(boolean suggest) { - request.suggest(suggest); - return this; - } - public IndicesStatsRequestBuilder setRequestCache(boolean requestCache) { request.requestCache(requestCache); return this; diff --git a/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java b/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java index c35a4cdbadb..bcabd7c5e12 100644 --- a/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java +++ b/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java @@ -51,6 +51,10 @@ public class SearchStats implements Streamable, ToXContent { private long scrollTimeInMillis; private long scrollCurrent; + private long suggestCount; + private long suggestTimeInMillis; + private long suggestCurrent; + Stats() { } @@ -58,7 +62,8 @@ public class SearchStats implements Streamable, ToXContent { public Stats( long queryCount, long queryTimeInMillis, long queryCurrent, long fetchCount, long fetchTimeInMillis, long fetchCurrent, - long scrollCount, long scrollTimeInMillis, long scrollCurrent + long scrollCount, long scrollTimeInMillis, long scrollCurrent, + long suggestCount, long suggestTimeInMillis, long suggestCurrent ) { this.queryCount = queryCount; this.queryTimeInMillis = queryTimeInMillis; @@ -71,13 +76,19 @@ public class SearchStats implements Streamable, ToXContent { this.scrollCount = scrollCount; this.scrollTimeInMillis = scrollTimeInMillis; this.scrollCurrent = scrollCurrent; + + this.suggestCount = suggestCount; + this.suggestTimeInMillis = suggestTimeInMillis; + this.suggestCurrent = suggestCurrent; + } public Stats(Stats stats) { this( stats.queryCount, stats.queryTimeInMillis, stats.queryCurrent, stats.fetchCount, stats.fetchTimeInMillis, stats.fetchCurrent, - stats.scrollCount, stats.scrollTimeInMillis, stats.scrollCurrent + stats.scrollCount, stats.scrollTimeInMillis, stats.scrollCurrent, + stats.suggestCount, stats.suggestTimeInMillis, stats.suggestCurrent ); } @@ -93,6 +104,10 @@ public class SearchStats implements Streamable, ToXContent { scrollCount += stats.scrollCount; scrollTimeInMillis += stats.scrollTimeInMillis; scrollCurrent += stats.scrollCurrent; + + suggestCount += stats.suggestCount; + suggestTimeInMillis += stats.suggestTimeInMillis; + suggestCurrent += stats.suggestCurrent; } public long getQueryCount() { @@ -143,6 +158,22 @@ public class SearchStats implements Streamable, ToXContent { return scrollCurrent; } + public long getSuggestCount() { + return suggestCount; + } + + public long getSuggestTimeInMillis() { + return suggestTimeInMillis; + } + + public TimeValue getSuggestTime() { + return new TimeValue(suggestTimeInMillis); + } + + public long getSuggestCurrent() { + return suggestCurrent; + } + public static Stats readStats(StreamInput in) throws IOException { Stats stats = new Stats(); stats.readFrom(in); @@ -162,6 +193,10 @@ public class SearchStats implements Streamable, ToXContent { scrollCount = in.readVLong(); scrollTimeInMillis = in.readVLong(); scrollCurrent = in.readVLong(); + + suggestCount = in.readVLong(); + suggestTimeInMillis = in.readVLong(); + suggestCurrent = in.readVLong(); } @Override @@ -177,6 +212,10 @@ public class SearchStats implements Streamable, ToXContent { out.writeVLong(scrollCount); out.writeVLong(scrollTimeInMillis); out.writeVLong(scrollCurrent); + + out.writeVLong(suggestCount); + out.writeVLong(suggestTimeInMillis); + out.writeVLong(suggestCurrent); } @Override @@ -193,6 +232,10 @@ public class SearchStats implements Streamable, ToXContent { builder.timeValueField(Fields.SCROLL_TIME_IN_MILLIS, Fields.SCROLL_TIME, scrollTimeInMillis); builder.field(Fields.SCROLL_CURRENT, scrollCurrent); + builder.field(Fields.SUGGEST_TOTAL, suggestCount); + builder.timeValueField(Fields.SUGGEST_TIME_IN_MILLIS, Fields.SUGGEST_TIME, suggestTimeInMillis); + builder.field(Fields.SUGGEST_CURRENT, suggestCurrent); + return builder; } } @@ -292,6 +335,10 @@ public class SearchStats implements Streamable, ToXContent { static final XContentBuilderString SCROLL_TIME = new XContentBuilderString("scroll_time"); static final XContentBuilderString SCROLL_TIME_IN_MILLIS = new XContentBuilderString("scroll_time_in_millis"); static final XContentBuilderString SCROLL_CURRENT = new XContentBuilderString("scroll_current"); + static final XContentBuilderString SUGGEST_TOTAL = new XContentBuilderString("suggest_total"); + static final XContentBuilderString SUGGEST_TIME = new XContentBuilderString("suggest_time"); + static final XContentBuilderString SUGGEST_TIME_IN_MILLIS = new XContentBuilderString("suggest_time_in_millis"); + static final XContentBuilderString SUGGEST_CURRENT = new XContentBuilderString("suggest_current"); } public static SearchStats readSearchStats(StreamInput in) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java b/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java index 1a155d17964..748bb01bd54 100644 --- a/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java +++ b/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java @@ -23,13 +23,13 @@ import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.metrics.MeanMetric; import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.SearchSlowLog; import org.elasticsearch.search.internal.SearchContext; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import static java.util.Collections.emptyMap; @@ -72,64 +72,51 @@ public final class ShardSearchStats { } public void onPreQueryPhase(SearchContext searchContext) { - totalStats.queryCurrent.inc(); - if (searchContext.groupStats() != null) { - for (int i = 0; i < searchContext.groupStats().size(); i++) { - groupStats(searchContext.groupStats().get(i)).queryCurrent.inc(); + computeStats(searchContext, statsHolder -> { + if (searchContext.hasOnlySuggest()) { + statsHolder.suggestCurrent.inc(); + } else { + statsHolder.queryCurrent.inc(); } - } + }); } public void onFailedQueryPhase(SearchContext searchContext) { - totalStats.queryCurrent.dec(); - if (searchContext.groupStats() != null) { - for (int i = 0; i < searchContext.groupStats().size(); i++) { - groupStats(searchContext.groupStats().get(i)).queryCurrent.dec(); + computeStats(searchContext, statsHolder -> { + if (searchContext.hasOnlySuggest()) { + statsHolder.suggestCurrent.dec(); + } else { + statsHolder.queryCurrent.dec(); } - } + }); } public void onQueryPhase(SearchContext searchContext, long tookInNanos) { - totalStats.queryMetric.inc(tookInNanos); - totalStats.queryCurrent.dec(); - if (searchContext.groupStats() != null) { - for (int i = 0; i < searchContext.groupStats().size(); i++) { - StatsHolder statsHolder = groupStats(searchContext.groupStats().get(i)); + computeStats(searchContext, statsHolder -> { + if (searchContext.hasOnlySuggest()) { + statsHolder.suggestMetric.inc(tookInNanos); + statsHolder.suggestCurrent.dec(); + } else { statsHolder.queryMetric.inc(tookInNanos); statsHolder.queryCurrent.dec(); } - } + }); slowLogSearchService.onQueryPhase(searchContext, tookInNanos); } public void onPreFetchPhase(SearchContext searchContext) { - totalStats.fetchCurrent.inc(); - if (searchContext.groupStats() != null) { - for (int i = 0; i < searchContext.groupStats().size(); i++) { - groupStats(searchContext.groupStats().get(i)).fetchCurrent.inc(); - } - } + computeStats(searchContext, statsHolder -> statsHolder.fetchCurrent.inc()); } public void onFailedFetchPhase(SearchContext searchContext) { - totalStats.fetchCurrent.dec(); - if (searchContext.groupStats() != null) { - for (int i = 0; i < searchContext.groupStats().size(); i++) { - groupStats(searchContext.groupStats().get(i)).fetchCurrent.dec(); - } - } + computeStats(searchContext, statsHolder -> statsHolder.fetchCurrent.dec()); } public void onFetchPhase(SearchContext searchContext, long tookInNanos) { - totalStats.fetchMetric.inc(tookInNanos); - totalStats.fetchCurrent.dec(); - if (searchContext.groupStats() != null) { - for (int i = 0; i < searchContext.groupStats().size(); i++) { - StatsHolder statsHolder = groupStats(searchContext.groupStats().get(i)); - statsHolder.fetchMetric.inc(tookInNanos); - statsHolder.fetchCurrent.dec(); - } - } + computeStats(searchContext, statsHolder -> { + statsHolder.fetchMetric.inc(tookInNanos); + statsHolder.fetchCurrent.dec(); + }); slowLogSearchService.onFetchPhase(searchContext, tookInNanos); } @@ -149,6 +136,15 @@ public final class ShardSearchStats { } } + private void computeStats(SearchContext searchContext, Consumer consumer) { + consumer.accept(totalStats); + if (searchContext.groupStats() != null) { + for (int i = 0; i < searchContext.groupStats().size(); i++) { + consumer.accept(groupStats(searchContext.groupStats().get(i))); + } + } + } + private StatsHolder groupStats(String group) { StatsHolder stats = groupsStats.get(group); if (stats == null) { @@ -184,26 +180,30 @@ public final class ShardSearchStats { public final MeanMetric queryMetric = new MeanMetric(); public final MeanMetric fetchMetric = new MeanMetric(); public final MeanMetric scrollMetric = new MeanMetric(); + public final MeanMetric suggestMetric = new MeanMetric(); public final CounterMetric queryCurrent = new CounterMetric(); public final CounterMetric fetchCurrent = new CounterMetric(); public final CounterMetric scrollCurrent = new CounterMetric(); + public final CounterMetric suggestCurrent = new CounterMetric(); public SearchStats.Stats stats() { return new SearchStats.Stats( queryMetric.count(), TimeUnit.NANOSECONDS.toMillis(queryMetric.sum()), queryCurrent.count(), fetchMetric.count(), TimeUnit.NANOSECONDS.toMillis(fetchMetric.sum()), fetchCurrent.count(), - scrollMetric.count(), TimeUnit.NANOSECONDS.toMillis(scrollMetric.sum()), scrollCurrent.count() + scrollMetric.count(), TimeUnit.NANOSECONDS.toMillis(scrollMetric.sum()), scrollCurrent.count(), + suggestMetric.count(), TimeUnit.NANOSECONDS.toMillis(suggestMetric.sum()), suggestCurrent.count() ); } public long totalCurrent() { - return queryCurrent.count() + fetchCurrent.count() + scrollCurrent.count(); + return queryCurrent.count() + fetchCurrent.count() + scrollCurrent.count() + suggestCurrent.count(); } public void clear() { queryMetric.clear(); fetchMetric.clear(); scrollMetric.clear(); + suggestMetric.clear(); } } } diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 7b19a12bbaa..b667a1de689 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -92,8 +92,6 @@ import org.elasticsearch.index.store.Store; import org.elasticsearch.index.store.Store.MetadataSnapshot; import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.index.store.StoreStats; -import org.elasticsearch.index.suggest.stats.ShardSuggestMetric; -import org.elasticsearch.index.suggest.stats.SuggestStats; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogConfig; import org.elasticsearch.index.translog.TranslogStats; @@ -135,7 +133,6 @@ public class IndexShard extends AbstractIndexShardComponent { private final ShardRequestCache shardQueryCache; private final ShardFieldData shardFieldData; private final IndexFieldDataService indexFieldDataService; - private final ShardSuggestMetric shardSuggestMetric = new ShardSuggestMetric(); private final ShardBitsetFilterCache shardBitsetFilterCache; private final Object mutex = new Object(); private final String checkIndexOnStartup; @@ -254,10 +251,6 @@ public class IndexShard extends AbstractIndexShardComponent { return this.getService; } - public ShardSuggestMetric getSuggestMetric() { - return shardSuggestMetric; - } - public ShardBitsetFilterCache shardBitsetFilterCache() { return shardBitsetFilterCache; } @@ -631,10 +624,6 @@ public class IndexShard extends AbstractIndexShardComponent { return getEngine().getTranslog().stats(); } - public SuggestStats suggestStats() { - return shardSuggestMetric.stats(); - } - public CompletionStats completionStats(String... fields) { CompletionStats completionStats = new CompletionStats(); try (final Engine.Searcher currentSearcher = acquireSearcher("completion_stats")) { diff --git a/core/src/main/java/org/elasticsearch/index/suggest/stats/ShardSuggestMetric.java b/core/src/main/java/org/elasticsearch/index/suggest/stats/ShardSuggestMetric.java deleted file mode 100644 index 750d7de7b22..00000000000 --- a/core/src/main/java/org/elasticsearch/index/suggest/stats/ShardSuggestMetric.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.suggest.stats; - -import org.elasticsearch.common.metrics.CounterMetric; -import org.elasticsearch.common.metrics.MeanMetric; - -import java.util.concurrent.TimeUnit; - -/** - * - */ -public final class ShardSuggestMetric { - private final MeanMetric suggestMetric = new MeanMetric(); - private final CounterMetric currentMetric = new CounterMetric(); - - /** - * Called before suggest - */ - public void preSuggest() { - currentMetric.inc(); - } - - /** - * Called after suggest - * @param tookInNanos time of suggest used in nanos - */ - public void postSuggest(long tookInNanos) { - currentMetric.dec(); - suggestMetric.inc(tookInNanos); - } - - /** - * @return The current stats - */ - public SuggestStats stats() { - return new SuggestStats(suggestMetric.count(), TimeUnit.NANOSECONDS.toMillis(suggestMetric.sum()), currentMetric.count()); - } -} diff --git a/core/src/main/java/org/elasticsearch/index/suggest/stats/SuggestStats.java b/core/src/main/java/org/elasticsearch/index/suggest/stats/SuggestStats.java deleted file mode 100644 index 1183a64833a..00000000000 --- a/core/src/main/java/org/elasticsearch/index/suggest/stats/SuggestStats.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.suggest.stats; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentBuilderString; - -import java.io.IOException; - -/** - * Exposes suggest related statistics. - */ -public class SuggestStats implements Streamable, ToXContent { - - private long suggestCount; - private long suggestTimeInMillis; - private long current; - - public SuggestStats() { - } - - SuggestStats(long suggestCount, long suggestTimeInMillis, long current) { - this.suggestCount = suggestCount; - this.suggestTimeInMillis = suggestTimeInMillis; - this.current = current; - } - - /** - * @return The number of times the suggest api has been invoked. - */ - public long getCount() { - return suggestCount; - } - - /** - * @return The total amount of time spend in the suggest api - */ - public long getTimeInMillis() { - return suggestTimeInMillis; - } - - /** - * @return The total amount of time spend in the suggest api - */ - public TimeValue getTime() { - return new TimeValue(getTimeInMillis()); - } - - /** - * @return The total amount of active suggest api invocations. - */ - public long getCurrent() { - return current; - } - - public void add(SuggestStats suggestStats) { - if (suggestStats != null) { - suggestCount += suggestStats.getCount(); - suggestTimeInMillis += suggestStats.getTimeInMillis(); - current += suggestStats.getCurrent(); - } - } - - public static SuggestStats readSuggestStats(StreamInput in) throws IOException { - SuggestStats stats = new SuggestStats(); - stats.readFrom(in); - return stats; - } - - static final class Fields { - static final XContentBuilderString SUGGEST = new XContentBuilderString("suggest"); - static final XContentBuilderString TOTAL = new XContentBuilderString("total"); - static final XContentBuilderString TIME = new XContentBuilderString("time"); - static final XContentBuilderString TIME_IN_MILLIS = new XContentBuilderString("time_in_millis"); - static final XContentBuilderString CURRENT = new XContentBuilderString("current"); - } - - - @Override - public void readFrom(StreamInput in) throws IOException { - suggestCount = in.readVLong(); - suggestTimeInMillis = in.readVLong(); - current = in.readVLong(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVLong(suggestCount); - out.writeVLong(suggestTimeInMillis); - out.writeVLong(current); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(Fields.SUGGEST); - builder.field(Fields.TOTAL, suggestCount); - builder.timeValueField(Fields.TIME_IN_MILLIS, Fields.TIME, suggestTimeInMillis); - builder.field(Fields.CURRENT, current); - builder.endObject(); - return builder; - } -} diff --git a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java index fce4e8411db..237975f2899 100644 --- a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java +++ b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java @@ -44,7 +44,6 @@ import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.store.StoreStats; -import org.elasticsearch.index.suggest.stats.SuggestStats; import org.elasticsearch.search.suggest.completion.CompletionStats; import java.io.IOException; @@ -149,11 +148,6 @@ public class NodeIndicesStats implements Streamable, ToXContent { return stats.getSegments(); } - @Nullable - public SuggestStats getSuggest() { - return stats.getSuggest(); - } - @Nullable public RecoveryStats getRecoveryStats() { return stats.getRecoveryStats(); diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java index 92fb21db38c..dbda83709ba 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java @@ -71,7 +71,7 @@ public class RestIndicesStatsAction extends BaseRestHandler { indicesStatsRequest.docs(metrics.contains("docs")); indicesStatsRequest.store(metrics.contains("store")); indicesStatsRequest.indexing(metrics.contains("indexing")); - indicesStatsRequest.search(metrics.contains("search")); + indicesStatsRequest.search(metrics.contains("search") || metrics.contains("suggest")); indicesStatsRequest.get(metrics.contains("get")); indicesStatsRequest.merge(metrics.contains("merge")); indicesStatsRequest.refresh(metrics.contains("refresh")); @@ -82,7 +82,6 @@ public class RestIndicesStatsAction extends BaseRestHandler { indicesStatsRequest.segments(metrics.contains("segments")); indicesStatsRequest.fieldData(metrics.contains("fielddata")); indicesStatsRequest.completion(metrics.contains("completion")); - indicesStatsRequest.suggest(metrics.contains("suggest")); indicesStatsRequest.requestCache(metrics.contains("request_cache")); indicesStatsRequest.recovery(metrics.contains("recovery")); indicesStatsRequest.translog(metrics.contains("translog")); diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java index 398ef9c15f7..958fa40b54b 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java @@ -490,14 +490,14 @@ public class RestIndicesAction extends AbstractCatAction { table.addCell(indexStats == null ? null : indexStats.getTotal().getWarmer().totalTime()); table.addCell(indexStats == null ? null : indexStats.getPrimaries().getWarmer().totalTime()); - table.addCell(indexStats == null ? null : indexStats.getTotal().getSuggest().getCurrent()); - table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSuggest().getCurrent()); + table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getSuggestCurrent()); + table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getSuggestCurrent()); - table.addCell(indexStats == null ? null : indexStats.getTotal().getSuggest().getTime()); - table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSuggest().getTime()); + table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getSuggestTime()); + table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getSuggestTime()); - table.addCell(indexStats == null ? null : indexStats.getTotal().getSuggest().getCount()); - table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSuggest().getCount()); + table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getSuggestCount()); + table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getSuggestCount()); table.addCell(indexStats == null ? null : indexStats.getTotal().getTotalMemory()); table.addCell(indexStats == null ? null : indexStats.getPrimaries().getTotalMemory()); diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java index c7b1550cbae..54ee855b543 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java @@ -49,7 +49,6 @@ import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.percolator.PercolatorQueryCacheStats; import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; -import org.elasticsearch.index.suggest.stats.SuggestStats; import org.elasticsearch.indices.NodeIndicesStats; import org.elasticsearch.monitor.fs.FsInfo; import org.elasticsearch.monitor.jvm.JvmInfo; @@ -362,10 +361,9 @@ public class RestNodesAction extends AbstractCatAction { table.addCell(segmentsStats == null ? null : segmentsStats.getVersionMapMemory()); table.addCell(segmentsStats == null ? null : segmentsStats.getBitsetMemory()); - SuggestStats suggestStats = indicesStats == null ? null : indicesStats.getSuggest(); - table.addCell(suggestStats == null ? null : suggestStats.getCurrent()); - table.addCell(suggestStats == null ? null : suggestStats.getTime()); - table.addCell(suggestStats == null ? null : suggestStats.getCount()); + table.addCell(searchStats == null ? null : searchStats.getTotal().getSuggestCurrent()); + table.addCell(searchStats == null ? null : searchStats.getTotal().getSuggestTime()); + table.addCell(searchStats == null ? null : searchStats.getTotal().getSuggestCount()); table.endRow(); } diff --git a/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java b/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java index 8711e89883f..62dec09e906 100644 --- a/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java +++ b/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java @@ -22,13 +22,15 @@ package org.elasticsearch.index.suggest.stats; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; -import org.elasticsearch.action.suggest.SuggestRequestBuilder; -import org.elasticsearch.action.suggest.SuggestResponse; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.search.stats.SearchStats; +import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; import org.elasticsearch.test.ESIntegTestCase; @@ -86,49 +88,50 @@ public class SuggestStatsIT extends ESIntegTestCase { long startTime = System.currentTimeMillis(); for (int i = 0; i < suggestAllIdx; i++) { - SuggestResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSuggest(), i).get(); + SearchResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSearch(), i).get(); assertAllSuccessful(suggestResponse); } for (int i = 0; i < suggestIdx1; i++) { - SuggestResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSuggest("test1"), i).get(); + SearchResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSearch("test1"), i).get(); assertAllSuccessful(suggestResponse); } for (int i = 0; i < suggestIdx2; i++) { - SuggestResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSuggest("test2"), i).get(); + SearchResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSearch("test2"), i).get(); assertAllSuccessful(suggestResponse); } long endTime = System.currentTimeMillis(); IndicesStatsResponse indicesStats = client().admin().indices().prepareStats().execute().actionGet(); + final SearchStats.Stats suggest = indicesStats.getTotal().getSearch().getTotal(); // check current - assertThat(indicesStats.getTotal().getSuggest().getCurrent(), equalTo(0L)); + assertThat(suggest.getSuggestCurrent(), equalTo(0L)); // check suggest count - assertThat(indicesStats.getTotal().getSuggest().getCount(), equalTo((long) (suggestAllIdx * totalShards + suggestIdx1 * shardsIdx1 + suggestIdx2 * shardsIdx2))); - assertThat(indicesStats.getIndices().get("test1").getTotal().getSuggest().getCount(), equalTo((long) ((suggestAllIdx + suggestIdx1) * shardsIdx1))); - assertThat(indicesStats.getIndices().get("test2").getTotal().getSuggest().getCount(), equalTo((long) ((suggestAllIdx + suggestIdx2) * shardsIdx2))); + assertThat(suggest.getSuggestCount(), equalTo((long) (suggestAllIdx * totalShards + suggestIdx1 * shardsIdx1 + suggestIdx2 * shardsIdx2))); + assertThat(indicesStats.getIndices().get("test1").getTotal().getSearch().getTotal().getSuggestCount(), equalTo((long) ((suggestAllIdx + suggestIdx1) * shardsIdx1))); + assertThat(indicesStats.getIndices().get("test2").getTotal().getSearch().getTotal().getSuggestCount(), equalTo((long) ((suggestAllIdx + suggestIdx2) * shardsIdx2))); logger.info("iter {}, iter1 {}, iter2 {}, {}", suggestAllIdx, suggestIdx1, suggestIdx2, endTime - startTime); // check suggest time - assertThat(indicesStats.getTotal().getSuggest().getTimeInMillis(), greaterThan(0L)); + assertThat(suggest.getSuggestTimeInMillis(), greaterThan(0L)); // the upperbound is num shards * total time since we do searches in parallel - assertThat(indicesStats.getTotal().getSuggest().getTimeInMillis(), lessThanOrEqualTo(totalShards * (endTime - startTime))); + assertThat(suggest.getSuggestTimeInMillis(), lessThanOrEqualTo(totalShards * (endTime - startTime))); NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().execute().actionGet(); NodeStats[] nodes = nodeStats.getNodes(); Set nodeIdsWithIndex = nodeIdsWithIndex("test1", "test2"); int num = 0; for (NodeStats stat : nodes) { - SuggestStats suggestStats = stat.getIndices().getSuggest(); + SearchStats.Stats suggestStats = stat.getIndices().getSearch().getTotal(); logger.info("evaluating {}", stat.getNode()); if (nodeIdsWithIndex.contains(stat.getNode().getId())) { - assertThat(suggestStats.getCount(), greaterThan(0L)); - assertThat(suggestStats.getTimeInMillis(), greaterThan(0L)); + assertThat(suggestStats.getSuggestCount(), greaterThan(0L)); + assertThat(suggestStats.getSuggestTimeInMillis(), greaterThan(0L)); num++; } else { - assertThat(suggestStats.getCount(), equalTo(0L)); - assertThat(suggestStats.getTimeInMillis(), equalTo(0L)); + assertThat(suggestStats.getSuggestCount(), equalTo(0L)); + assertThat(suggestStats.getSuggestTimeInMillis(), equalTo(0L)); } } @@ -136,15 +139,16 @@ public class SuggestStatsIT extends ESIntegTestCase { } - private SuggestRequestBuilder addSuggestions(SuggestRequestBuilder request, int i) { + private SearchRequestBuilder addSuggestions(SearchRequestBuilder request, int i) { + final SuggestBuilder suggestBuilder = new SuggestBuilder(); for (int s = 0; s < randomIntBetween(2, 10); s++) { if (randomBoolean()) { - request.addSuggestion("s" + s, new PhraseSuggestionBuilder("f").text("test" + i + " test" + (i - 1))); + suggestBuilder.addSuggestion("s" + s, new PhraseSuggestionBuilder("f").text("test" + i + " test" + (i - 1))); } else { - request.addSuggestion("s" + s, new TermSuggestionBuilder("f").text("test" + i)); + suggestBuilder.addSuggestion("s" + s, new TermSuggestionBuilder("f").text("test" + i)); } } - return request; + return request.suggest(suggestBuilder); } private Set nodeIdsWithIndex(String... indices) { diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java index 8ea053a64ab..046d0b4400f 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java @@ -44,7 +44,6 @@ import org.elasticsearch.action.search.MultiSearchRequestBuilder; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.suggest.SuggestRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Setting; @@ -53,6 +52,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.search.suggest.SuggestBuilders; import org.elasticsearch.test.ESIntegTestCase; @@ -94,7 +94,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists("test1", "test2"), true); verify(percolate("test1", "test2"), true); verify(mpercolate(null, "test1", "test2"), false); - verify(suggest("test1", "test2"), true); verify(getAliases("test1", "test2"), true); verify(getFieldMapping("test1", "test2"), true); verify(getMapping("test1", "test2"), true); @@ -114,7 +113,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists("test1", "test2").setIndicesOptions(options), true); verify(percolate("test1", "test2").setIndicesOptions(options), true); verify(mpercolate(options, "test1", "test2").setIndicesOptions(options), false); - verify(suggest("test1", "test2").setIndicesOptions(options), true); verify(getAliases("test1", "test2").setIndicesOptions(options), true); verify(getFieldMapping("test1", "test2").setIndicesOptions(options), true); verify(getMapping("test1", "test2").setIndicesOptions(options), true); @@ -134,7 +132,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists("test1", "test2").setIndicesOptions(options), false); verify(percolate("test1", "test2").setIndicesOptions(options), false); verify(mpercolate(options, "test1", "test2").setIndicesOptions(options), false); - verify(suggest("test1", "test2").setIndicesOptions(options), false); verify(getAliases("test1", "test2").setIndicesOptions(options), false); verify(getFieldMapping("test1", "test2").setIndicesOptions(options), false); verify(getMapping("test1", "test2").setIndicesOptions(options), false); @@ -156,7 +153,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists("test1", "test2").setIndicesOptions(options), false); verify(percolate("test1", "test2").setIndicesOptions(options), false); verify(mpercolate(options, "test1", "test2").setIndicesOptions(options), false); - verify(suggest("test1", "test2").setIndicesOptions(options), false); verify(getAliases("test1", "test2").setIndicesOptions(options), false); verify(getFieldMapping("test1", "test2").setIndicesOptions(options), false); verify(getMapping("test1", "test2").setIndicesOptions(options), false); @@ -186,7 +182,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists("test1").setIndicesOptions(options), true); verify(percolate("test1").setIndicesOptions(options), true); verify(mpercolate(options, "test1").setIndicesOptions(options), true); - verify(suggest("test1").setIndicesOptions(options), true); verify(getAliases("test1").setIndicesOptions(options), true); verify(getFieldMapping("test1").setIndicesOptions(options), true); verify(getMapping("test1").setIndicesOptions(options), true); @@ -206,7 +201,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists("test1").setIndicesOptions(options), false); verify(percolate("test1").setIndicesOptions(options), false); verify(mpercolate(options, "test1").setIndicesOptions(options), false); - verify(suggest("test1").setIndicesOptions(options), false); verify(getAliases("test1").setIndicesOptions(options), false); verify(getFieldMapping("test1").setIndicesOptions(options), false); verify(getMapping("test1").setIndicesOptions(options), false); @@ -229,7 +223,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists("test1").setIndicesOptions(options), false); verify(percolate("test1").setIndicesOptions(options), false); verify(mpercolate(options, "test1").setIndicesOptions(options), false); - verify(suggest("test1").setIndicesOptions(options), false); verify(getAliases("test1").setIndicesOptions(options), false); verify(getFieldMapping("test1").setIndicesOptions(options), false); verify(getMapping("test1").setIndicesOptions(options), false); @@ -250,7 +243,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(aliasExists("test1").setIndicesOptions(options), true); verify(typesExists("test1").setIndicesOptions(options), true); verify(percolate("test1").setIndicesOptions(options), true); - verify(suggest("test1").setIndicesOptions(options), true); verify(getAliases("test1").setIndicesOptions(options), true); verify(getFieldMapping("test1").setIndicesOptions(options), true); verify(getMapping("test1").setIndicesOptions(options), true); @@ -269,7 +261,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(aliasExists("test1").setIndicesOptions(options), false); verify(typesExists("test1").setIndicesOptions(options), false); verify(percolate("test1").setIndicesOptions(options), false); - verify(suggest("test1").setIndicesOptions(options), false); verify(getAliases("test1").setIndicesOptions(options), false); verify(getFieldMapping("test1").setIndicesOptions(options), false); verify(getMapping("test1").setIndicesOptions(options), false); @@ -291,7 +282,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(aliasExists("test1").setIndicesOptions(options), false); verify(typesExists("test1").setIndicesOptions(options), false); verify(percolate("test1").setIndicesOptions(options), false); - verify(suggest("test1").setIndicesOptions(options), false); verify(getAliases("test1").setIndicesOptions(options), false); verify(getFieldMapping("test1").setIndicesOptions(options), false); verify(getMapping("test1").setIndicesOptions(options), false); @@ -344,7 +334,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists(indices), false); verify(percolate(indices), false); verify(mpercolate(null, indices), false); - verify(suggest(indices), false); verify(getAliases(indices), false); verify(getFieldMapping(indices), false); verify(getMapping(indices), false); @@ -365,7 +354,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists(indices).setIndicesOptions(options), false); verify(percolate(indices).setIndicesOptions(options), false); verify(mpercolate(options, indices), false); - verify(suggest(indices).setIndicesOptions(options), false); verify(getAliases(indices).setIndicesOptions(options), false); verify(getFieldMapping(indices).setIndicesOptions(options), false); verify(getMapping(indices).setIndicesOptions(options), false); @@ -389,7 +377,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists(indices), false); verify(percolate(indices), false); verify(mpercolate(null, indices), false); - verify(suggest(indices), false); verify(getAliases(indices), false); verify(getFieldMapping(indices), false); verify(getMapping(indices), false); @@ -410,7 +397,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists(indices), false); verify(percolate(indices), false); verify(mpercolate(null, indices), false); - verify(suggest(indices), false); verify(getAliases(indices), false); verify(getFieldMapping(indices), false); verify(getMapping(indices), false); @@ -431,7 +417,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists(indices).setIndicesOptions(options), false); verify(percolate(indices).setIndicesOptions(options), false); verify(mpercolate(options, indices), false); - verify(suggest(indices).setIndicesOptions(options), false); verify(getAliases(indices).setIndicesOptions(options), false); verify(getFieldMapping(indices).setIndicesOptions(options), false); verify(getMapping(indices).setIndicesOptions(options), false); @@ -755,10 +740,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { return builder.add(percolate(indices)); } - private static SuggestRequestBuilder suggest(String... indices) { - return client().prepareSuggest(indices).addSuggestion("name", SuggestBuilders.termSuggestion("a")); - } - private static GetAliasesRequestBuilder getAliases(String... indices) { return client().admin().indices().prepareGetAliases("dummy").addIndices(indices); } diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java index b7cb64a7d2f..12e6d2b4799 100644 --- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -61,6 +61,7 @@ import java.util.Random; import static org.elasticsearch.cluster.metadata.IndexMetaData.PROTO; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.elasticsearch.common.xcontent.XContentFactory.contentBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; @@ -575,6 +576,10 @@ public class IndexStatsIT extends ESIntegTestCase { IndicesStatsResponse stats = builder.execute().actionGet(); for (Flag flag : values) { + if (flag == Flag.Suggest) { + // suggest flag is unused + continue; + } assertThat(isSet(flag, stats.getPrimaries()), equalTo(false)); assertThat(isSet(flag, stats.getTotal()), equalTo(false)); } @@ -610,6 +615,10 @@ public class IndexStatsIT extends ESIntegTestCase { } for (Flag flag : EnumSet.complementOf(flags)) { // check the complement + if (flag == Flag.Suggest) { + // suggest flag is unused + continue; + } assertThat(isSet(flag, stats.getPrimaries()), equalTo(false)); assertThat(isSet(flag, stats.getTotal()), equalTo(false)); } @@ -914,8 +923,7 @@ public class IndexStatsIT extends ESIntegTestCase { case Translog: builder.setTranslog(set); break; - case Suggest: - builder.setSuggest(set); + case Suggest: // unused break; case RequestCache: builder.setRequestCache(set); @@ -961,8 +969,8 @@ public class IndexStatsIT extends ESIntegTestCase { return response.getSegments() != null; case Translog: return response.getTranslog() != null; - case Suggest: - return response.getSuggest() != null; + case Suggest: // unused + return true; case RequestCache: return response.getRequestCache() != null; case Recovery: diff --git a/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java b/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java index c423513ac72..15fa7e64e3f 100644 --- a/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java +++ b/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java @@ -32,9 +32,9 @@ public class SearchStatsUnitTests extends ESTestCase { // let's create two dummy search stats with groups Map groupStats1 = new HashMap<>(); Map groupStats2 = new HashMap<>(); - groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1)); - SearchStats searchStats1 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats1); - SearchStats searchStats2 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats2); + groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)); + SearchStats searchStats1 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats1); + SearchStats searchStats2 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats2); // adding these two search stats and checking group stats are correct searchStats1.add(searchStats2); @@ -62,5 +62,8 @@ public class SearchStatsUnitTests extends ESTestCase { assertEquals(equalTo, stats.getScrollCount()); assertEquals(equalTo, stats.getScrollTimeInMillis()); assertEquals(equalTo, stats.getScrollCurrent()); + assertEquals(equalTo, stats.getSuggestCount()); + assertEquals(equalTo, stats.getSuggestTimeInMillis()); + assertEquals(equalTo, stats.getSuggestCurrent()); } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml index 19598c7363e..7b88ac57080 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml @@ -34,7 +34,6 @@ setup: - is_true: _all.total.completion - is_true: _all.total.segments - is_true: _all.total.translog - - is_true: _all.total.suggest - is_true: _all.total.recovery --- @@ -56,7 +55,6 @@ setup: - is_true: _all.total.completion - is_true: _all.total.segments - is_true: _all.total.translog - - is_true: _all.total.suggest - is_true: _all.total.recovery --- @@ -78,7 +76,6 @@ setup: - is_false: _all.total.completion - is_false: _all.total.segments - is_false: _all.total.translog - - is_false: _all.total.suggest - is_false: _all.total.recovery --- @@ -100,7 +97,6 @@ setup: - is_false: _all.total.completion - is_false: _all.total.segments - is_false: _all.total.translog - - is_false: _all.total.suggest - is_false: _all.total.recovery @@ -123,6 +119,5 @@ setup: - is_false: _all.total.completion - is_false: _all.total.segments - is_false: _all.total.translog - - is_false: _all.total.suggest - is_true: _all.total.recovery From 59513b308e8a694b9b97850d02578de719d39ae9 Mon Sep 17 00:00:00 2001 From: Areek Zillur Date: Wed, 23 Mar 2016 12:23:27 -0400 Subject: [PATCH 27/45] rename to isSuggestOnly --- .../java/org/elasticsearch/action/search/SearchRequest.java | 4 ++-- .../org/elasticsearch/search/builder/SearchSourceBuilder.java | 2 +- .../java/org/elasticsearch/search/internal/SearchContext.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 8a3099aa84c..e87d393bf05 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -298,8 +298,8 @@ public class SearchRequest extends ActionRequest implements Indic /** * @return true if the request only has suggest */ - public boolean hasOnlySuggest() { - return source != null && source.hasOnlySuggest(); + public boolean isSuggestOnly() { + return source != null && source.isSuggestOnly(); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 0093363d244..c83794ced90 100644 --- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -732,7 +732,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ /** * @return true if the source only has suggest */ - public boolean hasOnlySuggest() { + public boolean isSuggestOnly() { return suggestBuilder != null && queryBuilder == null && aggregations == null; } diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 65d41a85227..f9d0b6b9283 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -358,7 +358,7 @@ public abstract class SearchContext implements Releasable { */ public final boolean hasOnlySuggest() { return request().source() != null - && request().source().hasOnlySuggest(); + && request().source().isSuggestOnly(); } /** From de78621c95b71485552c2171f5785b70ea7fc2ac Mon Sep 17 00:00:00 2001 From: Areek Zillur Date: Wed, 23 Mar 2016 12:24:18 -0400 Subject: [PATCH 28/45] simplify handling suggest-only search request --- .../action/search/TransportSearchAction.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 33b3cd6085d..5b20a05bad5 100644 --- a/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -37,8 +37,6 @@ import org.elasticsearch.transport.TransportService; import java.util.Map; import java.util.Set; -import static org.elasticsearch.action.search.SearchType.DFS_QUERY_AND_FETCH; -import static org.elasticsearch.action.search.SearchType.DFS_QUERY_THEN_FETCH; import static org.elasticsearch.action.search.SearchType.QUERY_AND_FETCH; import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH; @@ -71,18 +69,20 @@ public class TransportSearchAction extends HandledTransportAction> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, searchRequest.routing(), searchRequest.indices()); int shardCount = clusterService.operationRouting().searchShardsCount(clusterState, concreteIndices, routingMap); - boolean hasOnlySuggest = searchRequest.hasOnlySuggest(); if (shardCount == 1) { // if we only have one group, then we always want Q_A_F, no need for DFS, and no need to do THEN since we hit one shard searchRequest.searchType(QUERY_AND_FETCH); - } else if (hasOnlySuggest && - (searchRequest.searchType() == DFS_QUERY_AND_FETCH || searchRequest.searchType() == DFS_QUERY_THEN_FETCH)) { - // convert to Q_T_F if we have only suggest - searchRequest.searchType(QUERY_THEN_FETCH); } - if (hasOnlySuggest && (searchRequest.requestCache() == null || searchRequest.requestCache())) { + if (searchRequest.isSuggestOnly()) { // disable request cache if we have only suggest searchRequest.requestCache(false); + switch (searchRequest.searchType()) { + case DFS_QUERY_AND_FETCH: + case DFS_QUERY_THEN_FETCH: + // convert to Q_T_F if we have only suggest + searchRequest.searchType(QUERY_THEN_FETCH); + break; + } } } catch (IndexNotFoundException | IndexClosedException e) { // ignore these failures, we will notify the search response if its really the case from the actual action From b1ceaaddf48255927f4aa34fbfba86107a7a2a4a Mon Sep 17 00:00:00 2001 From: Areek Zillur Date: Wed, 23 Mar 2016 14:37:07 -0400 Subject: [PATCH 29/45] remove dead code --- .../search/suggest/CompletionSuggestSearchIT.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java index e1f1dd02b15..152c450f6ec 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java @@ -896,9 +896,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { } public void assertSuggestions(String suggestionName, SuggestionBuilder suggestBuilder, String... suggestions) { - final SearchRequest searchRequest = Requests.searchRequest(INDEX); - searchRequest.source(new SearchSourceBuilder()); - searchRequest.source().suggest(); SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion(suggestionName, suggestBuilder)).execute().actionGet(); assertSuggestions(searchResponse, suggestionName, suggestions); } From e7e93f98e341be74a3d9c0a00643fed83cc09c2b Mon Sep 17 00:00:00 2001 From: Areek Zillur Date: Wed, 23 Mar 2016 14:38:53 -0400 Subject: [PATCH 30/45] add migration guide to use search api for suggest --- .../migration/migrate_5_0/java.asciidoc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/reference/migration/migrate_5_0/java.asciidoc b/docs/reference/migration/migrate_5_0/java.asciidoc index 59f0b1c15e2..df526740f33 100644 --- a/docs/reference/migration/migrate_5_0/java.asciidoc +++ b/docs/reference/migration/migrate_5_0/java.asciidoc @@ -22,6 +22,25 @@ can be replaced with client.prepareSearch(indices).setSource(new SearchSourceBuilder().size(0).query(query)).get(); ----- +==== Suggest api has been removed + +The suggest api has been removed from the Java api, use the suggest option in search api, it has been optimized +for suggest-only request. + +The following call + +[source,java] +----- +client.prepareSuggest(indices).addSuggestion("foo", SuggestBuilders.completionSuggestion("field").text("s")).get(); +----- + +can be replaced with + +[source,java] +----- +client.prepareSearch(indices).suggest(new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion("field").text("s"))).get(); +----- + ==== Elasticsearch will no longer detect logging implementations Elasticsearch now logs only to log4j 1.2. Previously if log4j wasn't on the From 442a6e00092df39d7545f9350dd95d3d6d06877c Mon Sep 17 00:00:00 2001 From: Areek Zillur Date: Wed, 23 Mar 2016 14:39:46 -0400 Subject: [PATCH 31/45] document suggest stats being merged with search stats --- docs/reference/indices/stats.asciidoc | 4 ++-- docs/reference/migration/migrate_5_0/index-apis.asciidoc | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/reference/indices/stats.asciidoc b/docs/reference/indices/stats.asciidoc index 87eda44e104..e990a7ff6bd 100644 --- a/docs/reference/indices/stats.asciidoc +++ b/docs/reference/indices/stats.asciidoc @@ -34,7 +34,8 @@ specified as well in the URI. Those stats can be any of: `get`:: Get statistics, including missing stats. -`search`:: Search statistics. You can include statistics for custom groups by adding +`search`:: Search statistics including suggest statistics. + You can include statistics for custom groups by adding an extra `groups` parameter (search operations can be associated with one or more groups). The `groups` parameter accepts a comma separated list of group names. Use `_all` to return statistics for all groups. @@ -47,7 +48,6 @@ specified as well in the URI. Those stats can be any of: `merge`:: Merge statistics. `request_cache`:: <> statistics. `refresh`:: Refresh statistics. -`suggest`:: Suggest statistics. `warmer`:: Warmer statistics. `translog`:: Translog statistics. diff --git a/docs/reference/migration/migrate_5_0/index-apis.asciidoc b/docs/reference/migration/migrate_5_0/index-apis.asciidoc index 72651295bbc..b820194c2d0 100644 --- a/docs/reference/migration/migrate_5_0/index-apis.asciidoc +++ b/docs/reference/migration/migrate_5_0/index-apis.asciidoc @@ -45,4 +45,7 @@ longer a double and is instead an object encapsulating the one-minute, five-minute and fifteen-minute load averages. Additionally, the recent CPU usage can be obtained from `OsStats.Cpu#getPercent`. +==== Suggest stats +Suggest stats exposed through `suggest` in indices stats has been merged +with `search` stats. `suggest` stats is exposed as part of `search` stats. From ad28fb9ec0c102df1ba7743b6bcb86037507afde Mon Sep 17 00:00:00 2001 From: debadair Date: Fri, 18 Mar 2016 14:07:43 -0700 Subject: [PATCH 32/45] Docs: Adding Painless to the Scripting documentation. --- docs/reference/modules.asciidoc | 9 +- docs/reference/modules/painless.asciidoc | 693 ++++++++++++++++++ docs/reference/modules/scripting.asciidoc | 1 - .../modules/scripting/scripting.asciidoc | 78 +- 4 files changed, 740 insertions(+), 41 deletions(-) create mode 100644 docs/reference/modules/painless.asciidoc diff --git a/docs/reference/modules.asciidoc b/docs/reference/modules.asciidoc index 5ef8a41d3f5..b71d1224e7e 100644 --- a/docs/reference/modules.asciidoc +++ b/docs/reference/modules.asciidoc @@ -45,6 +45,10 @@ The modules in this section are: <>:: A Java node client joins the cluster, but doesn't hold data or act as a master node. + +<>:: + + A built-in scripting language for Elasticsearch that's designed to be as secure as possible. <>:: @@ -53,7 +57,8 @@ The modules in this section are: <>:: Custom scripting available in Lucene Expressions, Groovy, Python, and - Javascript. + Javascript. You can also write scripts in the built-in scripting language, + <>. <>:: @@ -89,6 +94,8 @@ include::modules/network.asciidoc[] include::modules/node.asciidoc[] +include::modules/painless.asciidoc[] + include::modules/plugins.asciidoc[] include::modules/scripting.asciidoc[] diff --git a/docs/reference/modules/painless.asciidoc b/docs/reference/modules/painless.asciidoc new file mode 100644 index 00000000000..7c2bcdd9750 --- /dev/null +++ b/docs/reference/modules/painless.asciidoc @@ -0,0 +1,693 @@ +[[modules-scripting-painless]] +== Painless Scripting Language + +_Painless_ is a simple, secure scripting language built in to Elasticsearch as a module. +It is designed specifically for use with Elasticsearch and can safely be used dynamically. + +A Painless script is essentially a single function. Painless does not provide support +for defining multiple functions within a script. The Painless syntax is similar to +http://groovy-lang.org/index.html[Groovy]. + +You can use Painless anywhere a script can be used in Elasticsearch--simply set the `lang` parameter +to `painless`. + +[[painless-features]] +[float] +=== Painless Features + +* Control flow: `for` loops, `while` loops, `do/while` loops, `if/else` + +* Fully Typed: all available types/methods described in <> + +* Arithmetic operators: multiplication `*`, division `/`, addition `+`, subtraction `-`, precedence `( )` + +* Comparison operators: less than `<`, less than or equal to `<=`, greater than `>`, greater than or equal to `>=`, equal to `==`, and not equal to `!=`, reference equals `===`, reference not equals `!==` + +* Boolean operators: not `!`, and `&&`, or `||` + +* Bitwise operators: shift left `<<`, shift right `>>`, unsigned shift `>>>`, and `&`, or `|`, xor `^`, not `~` + +* Shortcuts for list, map access using the dot `.` operator + + +[[painless-examples]] +[float] +=== Painless Examples + +To illustrate how Painless works, let's load some hockey stats into an Elasticsearch index: + +[source,sh] +---------------------------------------------------------------- +curl -XDELETE http://localhost:9200/hockey-stats +curl -XPUT http://localhost:9200/hockey-stats +curl -XPUT http://localhost:9200/hockey-stats/player/1 -d '{"first":"johnny", "last":"gaudreau", "goals":[9, 27, 1], "assists":[17, 46, 0], "gp":[26, 82, 1]}' +curl -XPUT http://localhost:9200/hockey-stats/player/2 -d '{"first":"sean", "last":"monohan", "goals":[7, 54, 26], "assists":[11, 26, 13], "gp":[26, 82, 82]}' +curl -XPUT http://localhost:9200/hockey-stats/player/3 -d '{"first":"jiri", "last":"hudler", "goals":[5, 34, 36], "assists":[11, 62, 42], "gp":[24, 80, 79]}' +curl -XPUT http://localhost:9200/hockey-stats/player/4 -d '{"first":"micheal", "last":"frolik", "goals":[4, 6, 15], "assists":[8, 23, 15], "gp":[26, 82, 82]}' +curl -XPUT http://localhost:9200/hockey-stats/player/5 -d '{"first":"sam", "last":"bennett", "goals":[5, 0, 0], "assists":[8, 1, 0], "gp":[26, 1, 0]}' +curl -XPUT http://localhost:9200/hockey-stats/player/6 -d '{"first":"dennis", "last":"wideman", "goals":[0, 26, 15], "assists":[11, 30, 24], "gp":[26, 81, 82]}' +curl -XPUT http://localhost:9200/hockey-stats/player/7 -d '{"first":"david", "last":"jones", "goals":[7, 19, 5], "assists":[3, 17, 4], "gp":[26, 45, 34]}' +curl -XPUT http://localhost:9200/hockey-stats/player/8 -d '{"first":"tj", "last":"brodie", "goals":[2, 14, 7], "assists":[8, 42, 30], "gp":[26, 82, 82]}' +curl -XPUT http://localhost:9200/hockey-stats/player/9 -d '{"first":"mark", "last":"giordano", "goals":[6, 30, 15], "assists":[3, 30, 24], "gp":[26, 60, 63]}' +curl -XPUT http://localhost:9200/hockey-stats/player/10 -d '{"first":"mikael", "last":"backlund", "goals":[3, 15, 13], "assists":[6, 24, 18], "gp":[26, 82, 82]}' +curl -XPUT http://localhost:9200/hockey-stats/player/11 -d '{"first":"joe", "last":"colborne", "goals":[3, 18, 13], "assists":[6, 20, 24], "gp":[26, 67, 82]}' +---------------------------------------------------------------- + +[float] +==== Accessing Doc Values from Painless + +All Painless scripts take in a `Map` of values called `input`. Document values can be accessed through another `Map` within the `input` variable. + +For example, the following script calculates a player's total goals. This example uses a strongly typed `int` and a `for` loop. + +[source,sh] +---------------------------------------------------------------- +curl -XGET http://localhost:9200/hockey-stats/_search -d '{ + "query": { + "function_score": { + "script_score" : { + "script" : { + "inline": + "int total = 0; for (int i = 0; i < input.doc.goals.size(); ++i) { total += input.doc.goals[i]; } return total;", + "lang": "painless" + } + } + } + } +}' +---------------------------------------------------------------- + +Alternatively, you could do the same thing using a script field instead of a function score: + +[source,sh] +---------------------------------------------------------------- +curl -XGET http://localhost:9200/hockey-stats/_search -d '{ + "query": { + "match_all": {}}, + "script_fields": { + "total_goals": { + "script": { + "inline": "int total = 0; for (int i = 0; i < input.doc.goals.size(); ++i) { total += input.doc.goals[i]; } return total;", + "lang": "painless" + } + } + } +}' +---------------------------------------------------------------- + +You must always specify the index of the field value you want, even if there's only a single item in the field. +All fields in Elasticsearch are multi-valued and Painless does not provide a `.value` shortcut. The following example uses a Painless script to sort the players by their combined first and last names. The names are accessed using +`input.doc.first.0` and `input.doc.last.0`. + +[source,sh] +---------------------------------------------------------------- +curl -XGET http://localhost:9200/hockey-stats/_search -d '{ + "query" : { + "match_all": {}}, + "sort" : { + "_script" : { + "type" : "string", + "script" : {"inline": "input.doc.first.0 + \" \" + input.doc.last.0", + "lang": "painless"}, + "order" : "asc" + } + } +}' +---------------------------------------------------------------- + +[float] +==== Updating Fields with Painless + +You can also easily update fields. You access the original source for a field as `input.ctx._source.`. + +First, let's look at the source data for a player by submitting the following request: + +[source,sh] +---------------------------------------------------------------- +curl -XGET http://localhost:9200/hockey-stats/_search -d '{ + "fields" : ["_id", "_source"], "query" : { + "term" : { "_id" : 1 } + } +}' +---------------------------------------------------------------- + +To change player 1's last name to _hockey_, simply set `input.ctx._source.last` to the new value: + +[source,sh] +---------------------------------------------------------------- +curl -XPOST http://localhost:9200/hockey-stats/player/1/_update -d '{ + "script": { + "inline": "input.ctx._source.last = input.last", + "params": {"last": "hockey"}, + "lang": "painless" + } +}' +---------------------------------------------------------------- + +You can also add fields to a document. For example, this script adds a new field that contains +the player's nickname, _hockey_. + +[source,sh] +---------------------------------------------------------------- +curl -XPOST http://localhost:9200/hockey-stats/player/1/_update -d '{ + "script": { + "inline": "input.ctx._source.last = input.last input.ctx._source.nick = input.nick", + "params": {"last": "gaudreau", "nick": "hockey"}, + "lang": "painless" + } +}' +---------------------------------------------------------------- + +[float] +==== Writing Type-Safe Scripts to Improve Performance + +If you explicitly specify types, the compiler doesn't have to perform type lookups at runtime, which can significantly +improve performance. For example, the following script performs the same first name, last name sort we showed before, +but it's fully type-safe. + +[source,sh] +---------------------------------------------------------------- +curl -XGET http://localhost:9200/hockey-stats/_search -d '{ + "query": { + "match_all": {} + }, + "script_fields": { + "full_name_dynamic": { + "script": { + "inline": "def first = input.doc.first.0; def last = input.doc.last.0; return first + \" \" + last;", + "lang": "painless" + } + }, + "full_name_static": { + "script": { + "inline": + "String first = (String)((List)((Map)input.get(\"doc\")).get(\"first\")).get(0); String last = (String)((List)((Map)input.get(\"doc\")).get(\"last\")).get(0); return first + \" \" + last;", + "lang": "painless" + } + } + } +}' +---------------------------------------------------------------- + +[[painless-api]] +[float] +=== Painless API + +The following types are available for use in the Painless language. Most types and methods map directly to their Java equivalents--for more information, see the corresponding https://docs.oracle.com/javase/8/docs/api/java/lang/package-summary.html[Javadoc]. + + +[float] +==== Dynamic Types + +`def` (This type can be used to represent any other type.) + +[float] +==== Basic Types + +`void` + +`boolean` + +`short` + +`char` + +`int` + +`long` + +`float` + +`double` + +[float] +==== Complex Types + +Non-static methods/members in superclasses are available to subclasses. +Generic types with unspecified generic parameters are parameters of type `def`. + +----- +ArithmeticException extends Exception + () +----- + +----- +ArrayList extends List + () +----- + +----- +ArrayList extends List + () +----- + +----- +ArrayList extends List + () +----- + +----- +Boolean extends Object + (boolean) + static Boolean valueOf(boolean) + boolean booleanValue() +----- + +----- +Character extends Object + (char) + static Character valueOf(char) + char charValue() + static char MIN_VALUE + static char MAX_VALUE +----- + +----- +CharSequence extends Object + char charAt(int) + int length() +----- + +----- +Collection extends Object + boolean add(def) + void clear() + boolean contains(def) + boolean isEmpty() + Iterator iterator() + boolean remove(def) + int size() +----- + +----- +Collection extends Object + boolean add(Object) + void clear() + boolean contains(Object) + boolean isEmpty() + Iterator iterator() + boolean remove(Object) + int size() +----- + +----- +Collection extends Object + boolean add(String) + void clear() + boolean contains(String) + boolean isEmpty() + Iterator iterator() + boolean remove(String) + int size() +----- + +----- +Double extends Number + (double) + static Double valueOf(double) + static double MIN_VALUE + static double MAX_VALUE +----- + +----- +Exception extends Object + String getMessage() +----- + +----- +Float extends Number + (float) + static Float valueOf(float) + static float MIN_VALUE + static float MAX_VALUE +----- + +----- +HashMap extends Map + () +----- + +----- +HashMap extends Map + () +----- + +----- +HashMap extends Map + () +----- + +----- +HashMap extends Map + () +----- + +----- +IllegalArgument extends Exception + () +----- + +----- +IllegalState extends Exception + () +----- + +----- +Integer extends Number + (int) + static Integer valueOf(int) + static int MIN_VALUE + static int MAX_VALUE +----- + +----- +Iterator extends Object + boolean hasNext() + def next() + void remove() +----- + +----- +Iterator extends Object + boolean hasNext() + String next() + void remove() +----- + +----- +List extends Collection + def set(int, def) + def get(int) + def remove(int) +----- + +----- +List extends Collection + Object set(int, Object) + Object get(int) + Object remove(int) +----- + +----- +List extends Collection + String set(int, String) + String get(int) + String remove(int) +----- + +----- +Long extends Number + (long) + static Long valueOf(long) + static long MIN_VALUE + static long MAX_VALUE +----- + +----- +Map extends Object + def put (def, def) + def get (def) + def remove (def) + boolean isEmpty() + int size() + boolean containsKey(def) + boolean containsValue(def) + Set keySet() + Collection values() +----- + +----- +Map extends Object + Object put (Object, Object) + Object get (Object) + Object remove (Object) + boolean isEmpty() + int size() + boolean containsKey(Object) + boolean containsValue(Object) + Set keySet() + Collection values() +----- + +----- +Map extends Object + def put (String, def) + def get (String) + def remove (String) + boolean isEmpty() + int size() + boolean containsKey(String) + boolean containsValue(def) + Set keySet() + Collection values() +----- + +----- +Map extends Object + Object put (String, Object) + Object get (String) + Object remove (String) + boolean isEmpty() + int size() + boolean containsKey(String) + boolean containsValue(Object) + Set keySet() + Collection values() +----- + +----- +Number extends Object + short shortValue() + short shortValue() + int intValue() + long longValue() + float floatValue() + double doubleValue() +----- + +----- +Object + String toString() + boolean equals(Object) + int hashCode() +----- + +----- +Set extends Collection +----- + +----- +Set extends Collection +----- + +----- +Set extends Collection +----- + +----- +Short extends Number + (short) + static Short valueOf(short) + static short MIN_VALUE + static short MAX_VALUE +----- + +----- +String extends CharSequence + (String) + int codePointAt(int) + int compareTo(String) + String concat(String) + boolean endsWith(String) + int indexOf(String, int) + boolean isEmpty() + String replace(CharSequence, CharSequence) + boolean startsWith(String) + String substring(int, int) + char[] toCharArray() + String trim() +----- + +----- +NumberFormatException extends Exception + () +----- + +----- +Void extends Object +----- + +[float] +==== Utility Classes + +----- +Math + static double abs(double) + static float fabs(float) + static long labs(long) + static int iabs(int) + static double acos(double) + static double asin(double) + static double atan(double) + static double atan2(double) + static double cbrt(double) + static double ceil(double) + static double cos(double) + static double cosh(double) + static double exp(double) + static double expm1(double) + static double floor(double) + static double hypt(double, double) + static double abs(double) + static double log(double) + static double log10(double) + static double log1p(double) + static double max(double, double) + static float fmax(float, float) + static long lmax(long, long) + static int imax(int, int) + static double min(double, double) + static float fmin(float, float) + static long lmin(long, long) + static int imin(int, int) + static double pow(double, double) + static double random() + static double rint(double) + static long round(double) + static double sin(double) + static double sinh(double) + static double sqrt(double) + static double tan(double) + static double tanh(double) + static double toDegrees(double) + static double toRadians(double) +----- + +----- +Utility + static boolean NumberToboolean(Number) + static char NumberTochar(Number) + static Boolean NumberToBoolean(Number) + static Short NumberToShort(Number) + static Character NumberToCharacter(Number) + static Integer NumberToInteger(Number) + static Long NumberToLong(Number) + static Float NumberToFloat(Number) + static Double NumberToDouble(Number) + static byte booleanTobyte(boolean) + static short booleanToshort(boolean) + static char booleanTochar(boolean) + static int booleanToint(boolean) + static long booleanTolong(boolean) + static float booleanTofloat(boolean) + static double booleanTodouble(boolean) + static Integer booleanToInteger(boolean) + static byte BooleanTobyte(Boolean) + static short BooleanToshort(Boolean) + static char BooleanTochar(Boolean) + static int BooleanToint(Boolean) + static long BooleanTolong(Boolean) + static float BooleanTofloat(Boolean) + static double BooleanTodouble(Boolean) + static Byte BooleanToByte(Boolean) + static Short BooleanToShort(Boolean) + static Character BooleanToCharacter(Boolean) + static Integer BooleanToInteger(Boolean) + static Long BooleanToLong(Boolean) + static Float BooleanToFloat(Boolean) + static Double BooleanToDouble(Boolean) + static boolean byteToboolean(byte) + static Short byteToShort(byte) + static Character byteToCharacter(byte) + static Integer byteToInteger(byte) + static Long byteToLong(byte) + static Float byteToFloat(byte) + static Double byteToDouble(byte) + static boolean ByteToboolean(Byte) + static char ByteTochar(Byte) + static boolean shortToboolean(short) + static Byte shortToByte(short) + static Character shortToCharacter(short) + static Integer shortToInteger(short) + static Long shortToLong(short) + static Float shortToFloat(short) + static Double shortToDouble(short) + static boolean ShortToboolean(Short) + static char ShortTochar(Short) + static boolean charToboolean(char) + static Byte charToByte(char) + static Short charToShort(char) + static Integer charToInteger(char) + static Long charToLong(char) + static Float charToFloat(char) + static Double charToDouble(char) + static boolean CharacterToboolean(Character) + static byte CharacterTobyte(Character) + static short CharacterToshort(Character) + static int CharacterToint(Character) + static long CharacterTolong(Character) + static float CharacterTofloat(Character) + static double CharacterTodouble(Character) + static Boolean CharacterToBoolean(Character) + static Byte CharacterToByte(Character) + static Short CharacterToShort(Character) + static Integer CharacterToInteger(Character) + static Long CharacterToLong(Character) + static Float CharacterToFloat(Character) + static Double CharacterToDouble(Character) + static boolean intToboolean(int) + static Byte intToByte(int) + static Short intToShort(int) + static Character intToCharacter(int) + static Long intToLong(int) + static Float intToFloat(int) + static Double intToDouble(int) + static boolean IntegerToboolean(Integer) + static char IntegerTochar(Integer) + static boolean longToboolean(long) + static Byte longToByte(long) + static Short longToShort(long) + static Character longToCharacter(long) + static Integer longToInteger(long) + static Float longToFloat(long) + static Double longToDouble(long) + static boolean LongToboolean(Long) + static char LongTochar(Long) + static boolean floatToboolean(float) + static Byte floatToByte(float) + static Short floatToShort(float) + static Character floatToCharacter(float) + static Integer floatToInteger(float) + static Long floatToLong(float) + static Double floatToDouble(float) + static boolean FloatToboolean(Float) + static char FloatTochar(Float) + static boolean doubleToboolean(double) + static Byte doubleToByte(double) + static Short doubleToShort(double) + static Character doubleToCharacter(double) + static Integer doubleToInteger(double) + static Long doubleToLong(double) + static Float doubleToFloat(double) + static boolean DoubleToboolean(Double) + static char DoubleTochar(Double) +----- + +----- +Def + static boolean defToboolean(def) + static byte defTobyte(def) + static short defToshort(def) + static char defTochar(def) + static int defToint(def) + static long defTolong(def) + static float defTofloat(def) + static double defTodouble(def) + static Boolean defToBoolean(def) + static Byte defToByte(def) + static Character defToCharacter(def) + static Integer defToInteger(def) + static Long defToLong(def) + static Float defToFloat(def) + static Double defToDouble(def) +----- \ No newline at end of file diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/modules/scripting.asciidoc index f4374a0f9b3..114740306ee 100644 --- a/docs/reference/modules/scripting.asciidoc +++ b/docs/reference/modules/scripting.asciidoc @@ -3,4 +3,3 @@ include::scripting/scripting.asciidoc[] include::scripting/advanced-scripting.asciidoc[] include::scripting/security.asciidoc[] - diff --git a/docs/reference/modules/scripting/scripting.asciidoc b/docs/reference/modules/scripting/scripting.asciidoc index 4f9d84f34f8..12524407900 100644 --- a/docs/reference/modules/scripting/scripting.asciidoc +++ b/docs/reference/modules/scripting/scripting.asciidoc @@ -1,27 +1,48 @@ [[modules-scripting]] == Scripting -The scripting module allows to use scripts in order to evaluate custom -expressions. For example, scripts can be used to return "script fields" -as part of a search request, or can be used to evaluate a custom score -for a query and so on. +The scripting module enables you to use scripts to evaluate custom +expressions. For example, you could use a script to return "script fields" +as part of a search request or evaluate a custom score for a query. -The scripting module uses by default http://groovy-lang.org/[groovy] -(previously http://mvel.codehaus.org/[mvel] in 1.3.x and earlier) as the -scripting language with some extensions. Groovy is used since it is extremely -fast and very simple to use. +TIP: Elasticsearch now has a built-in scripting language called _Painless_ +that provides a more secure alternative for implementing +scripts for Elasticsearch. We encourage you to try it out-- +for more information, see <>. + +The default scripting language is http://groovy-lang.org/[groovy] +(http://mvel.codehaus.org/[mvel] was the default in 1.3.x and earlier). + +Additional `lang` plugins enable you to run scripts written in other languages. +Everywhere a script can be used, you can include a `lang` parameter +to specify the language of the script. Plugins are available for following languages: + +[cols="<,<,<",options="header",] +|======================================================================= +|Language |Sandboxed |Required plugin +|groovy |no |built-in +|expression |yes |built-in +|mustache |yes |built-in +/painless /yes /built-in (module) +|javascript |no |{plugins}/lang-javascript.html[elasticsearch-lang-javascript] +|python |no |{plugins}/lang-python.html[elasticsearch-lang-python] +|======================================================================= .Groovy dynamic scripting off by default from v1.4.3 [IMPORTANT] =================================================== -Groovy dynamic scripting is off by default, preventing dynamic Groovy scripts -from being accepted as part of a request or retrieved from the special -`.scripts` index. You will still be able to use Groovy scripts stored in files -in the `config/scripts/` directory on every node. +Groovy dynamic scripting is off by default. This prevents Groovy scripts +from being accepted as part of a request or retrieved from the +`.scripts` index. You can still use Groovy file scripts stored in +the `config/scripts/` directory on every node. -To convert an inline script to a file, take this simple script -as an example: +To convert an inline script to a file-based script, save the contents +of the `inline` field to a file with the `.groovy` extension and +store it in the `config/scripts` directory on every data node in your +cluster. + +For example, if you have the following inline script: [source,js] ----------------------------------- @@ -38,15 +59,9 @@ GET /_search } ----------------------------------- -Save the contents of the `inline` field as a file called `config/scripts/my_script.groovy` -on every data node in the cluster: +Save `1 + my_var` in a file called `config/scripts/my_script.groovy`. -[source,js] ------------------------------------ -1 + my_var ------------------------------------ - -Now you can access the script by file name (without the extension): +To use the script in a request, specify its name (without the `.groovy` extension) in the `file` field: [source,js] ----------------------------------- @@ -67,21 +82,8 @@ GET /_search =================================================== - -Additional `lang` plugins are provided to allow to execute scripts in -different languages. All places where a script can be used, a `lang` parameter -can be provided to define the language of the script. The following are the -supported scripting languages: - -[cols="<,<,<",options="header",] -|======================================================================= -|Language |Sandboxed |Required plugin -|groovy |no |built-in -|expression |yes |built-in -|mustache |yes |built-in -|javascript |no |{plugins}/lang-javascript.html[elasticsearch-lang-javascript] -|python |no |{plugins}/lang-python.html[elasticsearch-lang-python] -|======================================================================= +[float] +=== File-based Scripts To increase security, Elasticsearch does not allow you to specify scripts for non-sandboxed languages with a request. Instead, scripts must be placed in the @@ -219,8 +221,6 @@ Indexed scripts can be deleted by: curl -XDELETE localhost:9200/_scripts/groovy/indexedCalculateScore ----------------------------------- - - [float] [[enable-dynamic-scripting]] === Enabling dynamic scripting From 04e6e6c3e052b5c8284bb406dee3cba319b9b27e Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 23 Mar 2016 17:06:45 -0400 Subject: [PATCH 33/45] Fix cluster health API call --- .../test/resources/packaging/scripts/packaging_test_utils.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash index e2a072273bf..3d449ebf901 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash +++ b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash @@ -402,7 +402,7 @@ wait_for_elasticsearch_status() { curl -sS "http://localhost:9200/_cluster/health?wait_for_status=$desiredStatus&timeout=60s&pretty" else echo "Trying to connect to elasticsearch and wait for expected status $desiredStatus for index $index" - curl -sS "http://localhost:9200/_cluster/$index/health?wait_for_status=$desiredStatus&timeout=60s&pretty" + curl -sS "http://localhost:9200/_cluster/health/$index?wait_for_status=$desiredStatus&timeout=60s&pretty" fi if [ $? -eq 0 ]; then echo "Connected" From e16e113691e2bfaba6a075f835f8fd22a62fd0e8 Mon Sep 17 00:00:00 2001 From: Areek Zillur Date: Wed, 23 Mar 2016 17:53:49 -0400 Subject: [PATCH 34/45] Remove suggest threadpool In #17198, we removed suggest transport action, which used the `suggest` threadpool to execute requests. Now `suggest` threadpool is unused and suggest requests are executed on the `search` threadpool. --- .../rest/action/cat/RestThreadPoolAction.java | 2 -- .../java/org/elasticsearch/threadpool/ThreadPool.java | 3 --- .../elasticsearch/threadpool/ThreadPoolStatsTests.java | 2 -- docs/reference/migration/migrate_5_0/settings.asciidoc | 4 ++++ docs/reference/modules/threadpool.asciidoc | 7 +------ .../rest-api-spec/test/cat.thread_pool/10_basic.yaml | 10 ---------- 6 files changed, 5 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java index a0812f3e9a4..b4ba4bbeffa 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java @@ -66,7 +66,6 @@ public class RestThreadPoolAction extends AbstractCatAction { ThreadPool.Names.REFRESH, ThreadPool.Names.SEARCH, ThreadPool.Names.SNAPSHOT, - ThreadPool.Names.SUGGEST, ThreadPool.Names.WARMER }; @@ -81,7 +80,6 @@ public class RestThreadPoolAction extends AbstractCatAction { "r", "s", "sn", - "su", "w" }; diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index a04594e706a..b223b477740 100644 --- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -82,7 +82,6 @@ public class ThreadPool extends AbstractComponent implements Closeable { public static final String INDEX = "index"; public static final String BULK = "bulk"; public static final String SEARCH = "search"; - public static final String SUGGEST = "suggest"; public static final String MANAGEMENT = "management"; public static final String FLUSH = "flush"; public static final String REFRESH = "refresh"; @@ -139,7 +138,6 @@ public class ThreadPool extends AbstractComponent implements Closeable { map.put(Names.INDEX, ThreadPoolType.FIXED); map.put(Names.BULK, ThreadPoolType.FIXED); map.put(Names.SEARCH, ThreadPoolType.FIXED); - map.put(Names.SUGGEST, ThreadPoolType.FIXED); map.put(Names.MANAGEMENT, ThreadPoolType.SCALING); map.put(Names.FLUSH, ThreadPoolType.SCALING); map.put(Names.REFRESH, ThreadPoolType.SCALING); @@ -227,7 +225,6 @@ public class ThreadPool extends AbstractComponent implements Closeable { add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.BULK).size(availableProcessors).queueSize(50)); add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.GET).size(availableProcessors).queueSize(1000)); add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.SEARCH).size(((availableProcessors * 3) / 2) + 1).queueSize(1000)); - add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.SUGGEST).size(availableProcessors).queueSize(1000)); add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.MANAGEMENT).size(5).keepAlive("5m")); // no queue as this means clients will need to handle rejections on listener queue even if the operation succeeded // the assumption here is that the listeners should be very lightweight on the listeners side diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java index 25b0e06c4f1..fc0a8abac27 100644 --- a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java +++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java @@ -65,7 +65,6 @@ public class ThreadPoolStatsTests extends ESTestCase { try (BytesStreamOutput os = new BytesStreamOutput()) { List stats = new ArrayList<>(); - stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.SUGGEST, -1, 0, 0, 0, 0, 0L)); stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.SEARCH, -1, 0, 0, 0, 0, 0L)); stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.WARMER, -1, 0, 0, 0, 0, 0L)); stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.GENERIC, -1, 0, 0, 0, 0, 0L)); @@ -104,7 +103,6 @@ public class ThreadPoolStatsTests extends ESTestCase { ThreadPool.Names.GENERIC, ThreadPool.Names.SAME, ThreadPool.Names.SEARCH, - ThreadPool.Names.SUGGEST, ThreadPool.Names.WARMER)); } } diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc index bf242a9929f..f9b104e2fcf 100644 --- a/docs/reference/migration/migrate_5_0/settings.asciidoc +++ b/docs/reference/migration/migrate_5_0/settings.asciidoc @@ -71,6 +71,10 @@ type for any thread pool has been removed. It is still possible to adjust relevant thread pool parameters for each of the thread pools (e.g., depending on the thread pool type, `keep_alive`, `queue_size`, etc.). +==== Threadpool settings + +The `suggest` threadpool has been removed, now suggest requests use the +`search` threadpool. ==== Analysis settings diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc index bfd5474183c..199b6a9b88c 100644 --- a/docs/reference/modules/threadpool.asciidoc +++ b/docs/reference/modules/threadpool.asciidoc @@ -19,15 +19,10 @@ There are several thread pools, but the important ones include: queue_size of `200`. `search`:: - For count/search operations. Thread pool type is `fixed` + For count/search/suggest operations. Thread pool type is `fixed` with a size of `int((# of available_processors * 3) / 2) + 1`, queue_size of `1000`. -`suggest`:: - For suggest operations. Thread pool type is `fixed` - with a size of `# of available processors`, - queue_size of `1000`. - `get`:: For get operations. Thread pool type is `fixed` with a size of `# of available processors`, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml index 780edacd7b6..233e21bf470 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml @@ -138,16 +138,6 @@ /^ id \s+ snapshot.type \s+ snapshot.active \s+ snapshot.size \s+ snapshot.queue \s+ snapshot.queueSize \s+ snapshot.rejected \s+ snapshot.largest \s+ snapshot.completed \s+ snapshot.min \s+ snapshot.max \s+ snapshot.keepAlive \n (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - - do: - cat.thread_pool: - h: id,suggest.type,suggest.active,suggest.size,suggest.queue,suggest.queueSize,suggest.rejected,suggest.largest,suggest.completed,suggest.min,suggest.max,suggest.keepAlive - v: true - - - match: - $body: | - /^ id \s+ suggest.type \s+ suggest.active \s+ suggest.size \s+ suggest.queue \s+ suggest.queueSize \s+ suggest.rejected \s+ suggest.largest \s+ suggest.completed \s+ suggest.min \s+ suggest.max \s+ suggest.keepAlive \n - (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - - do: cat.thread_pool: h: id,warmer.type,warmer.active,warmer.size,warmer.queue,warmer.queueSize,warmer.rejected,warmer.largest,warmer.completed,warmer.min,warmer.max,warmer.keepAlive From aaa4d57fff41c3ac2bef16a8b42fb7d4cd8821d3 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 23 Mar 2016 16:13:11 -0400 Subject: [PATCH 35/45] [reindex] Don't attempt to refresh on noop If the user asks for a refresh but their reindex or update-by-query operation touched no indexes we should just skip the resfresh call entirely. Without this commit we refresh *all* indexes which is totally wrong. Closes #17296 --- .../AbstractAsyncBulkByScrollAction.java | 2 +- .../reindex/AsyncBulkByScrollActionTests.java | 20 ++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java index a1564aa136a..84b7ab58ef2 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -354,7 +354,7 @@ public abstract class AbstractAsyncBulkByScrollAction indexingFailures, List searchFailures, boolean timedOut) { - if (task.isCancelled() || false == mainRequest.isRefresh()) { + if (task.isCancelled() || false == mainRequest.isRefresh() || destinationIndices.isEmpty()) { finishHim(null, indexingFailures, searchFailures, timedOut); return; } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java index daf9a15daea..06f5226a7f4 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -458,23 +458,29 @@ public class AsyncBulkByScrollActionTests extends ESTestCase { } public void testRefreshIsFalseByDefault() throws Exception { - refreshTestCase(null, false); + refreshTestCase(null, true, false); } - public void testRefreshFalseDoesntMakeVisible() throws Exception { - refreshTestCase(false, false); + public void testRefreshFalseDoesntExecuteRefresh() throws Exception { + refreshTestCase(false, true, false); } - public void testRefreshTrueMakesVisible() throws Exception { - refreshTestCase(true, true); + public void testRefreshTrueExecutesRefresh() throws Exception { + refreshTestCase(true, true, true); } - private void refreshTestCase(Boolean refresh, boolean shouldRefresh) { + public void testRefreshTrueSkipsRefreshIfNoDestinationIndexes() throws Exception { + refreshTestCase(true, false, false); + } + + private void refreshTestCase(Boolean refresh, boolean addDestinationIndexes, boolean shouldRefresh) { if (refresh != null) { mainRequest.setRefresh(refresh); } DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction(); - action.addDestinationIndices(singleton("foo")); + if (addDestinationIndexes) { + action.addDestinationIndices(singleton("foo")); + } action.startNormalTermination(emptyList(), emptyList(), false); if (shouldRefresh) { assertArrayEquals(new String[] {"foo"}, client.lastRefreshRequest.get().indices()); From 201fc06f8d7e28a0940365314a786ac6230f7ace Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 23 Mar 2016 17:12:30 -0400 Subject: [PATCH 36/45] Fix TaskId#isSet to return true when id is set and not other way around During refactoring the name was changed, but the logic wasn't. This commit fixes the logic to match the name. --- .../node/tasks/cancel/TransportCancelTasksAction.java | 2 +- .../action/admin/cluster/node/tasks/list/TaskInfo.java | 2 +- .../action/support/tasks/BaseTasksRequest.java | 6 +++--- .../action/support/tasks/TransportTasksAction.java | 6 +++--- core/src/main/java/org/elasticsearch/tasks/TaskId.java | 6 +++--- core/src/main/java/org/elasticsearch/tasks/TaskManager.java | 4 ++-- .../action/admin/cluster/node/tasks/TasksIT.java | 6 +++--- .../admin/cluster/node/tasks/TransportTasksActionTests.java | 2 +- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java index 336f4c84596..9dbe4ee1aeb 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java @@ -84,7 +84,7 @@ public class TransportCancelTasksAction extends TransportTasksAction operation) { - if (request.getTaskId().isSet() == false) { + if (request.getTaskId().isSet()) { // we are only checking one task, we can optimize it CancellableTask task = taskManager.getCancellableTask(request.getTaskId().getId()); if (task != null) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java index c027bfa7ab4..ad9fa9509e8 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java @@ -178,7 +178,7 @@ public class TaskInfo implements Writeable, ToXContent { } builder.dateValueField("start_time_in_millis", "start_time", startTime); builder.timeValueField("running_time_in_nanos", "running_time", runningTimeNanos, TimeUnit.NANOSECONDS); - if (parentTaskId.isSet() == false) { + if (parentTaskId.isSet()) { builder.field("parent_task_id", parentTaskId.toString()); } return builder; diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java b/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java index f1045387259..dc296a84720 100644 --- a/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java +++ b/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java @@ -60,7 +60,7 @@ public class BaseTasksRequest> extends @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; - if (taskId.isSet() == false && nodesIds.length > 0) { + if (taskId.isSet() && nodesIds.length > 0) { validationException = addValidationError("task id cannot be used together with node ids", validationException); } @@ -165,12 +165,12 @@ public class BaseTasksRequest> extends if (getActions() != null && getActions().length > 0 && Regex.simpleMatch(getActions(), task.getAction()) == false) { return false; } - if (getTaskId().isSet() == false) { + if (getTaskId().isSet()) { if(getTaskId().getId() != task.getId()) { return false; } } - if (parentTaskId.isSet() == false) { + if (parentTaskId.isSet()) { if (parentTaskId.equals(task.getParentTaskId()) == false) { return false; } diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java index a14c6e00e14..97678e6c060 100644 --- a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java @@ -125,14 +125,14 @@ public abstract class TransportTasksAction< protected String[] resolveNodes(TasksRequest request, ClusterState clusterState) { if (request.getTaskId().isSet()) { - return clusterState.nodes().resolveNodesIds(request.getNodesIds()); - } else { return new String[]{request.getTaskId().getNodeId()}; + } else { + return clusterState.nodes().resolveNodesIds(request.getNodesIds()); } } protected void processTasks(TasksRequest request, Consumer operation) { - if (request.getTaskId().isSet() == false) { + if (request.getTaskId().isSet()) { // we are only checking one task, we can optimize it Task task = taskManager.getTask(request.getTaskId().getId()); if (task != null) { diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskId.java b/core/src/main/java/org/elasticsearch/tasks/TaskId.java index 5c5ad36cc17..d1f29e65226 100644 --- a/core/src/main/java/org/elasticsearch/tasks/TaskId.java +++ b/core/src/main/java/org/elasticsearch/tasks/TaskId.java @@ -73,15 +73,15 @@ public final class TaskId implements Writeable { } public boolean isSet() { - return id == -1L; + return id != -1L; } @Override public String toString() { if (isSet()) { - return "unset"; - } else { return nodeId + ":" + id; + } else { + return "unset"; } } diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java index 0c785573c99..c881e1756e8 100644 --- a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java +++ b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java @@ -76,7 +76,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen CancellableTaskHolder oldHolder = cancellableTasks.put(task.getId(), holder); assert oldHolder == null; // Check if this task was banned before we start it - if (task.getParentTaskId().isSet() == false && banedParents.isEmpty() == false) { + if (task.getParentTaskId().isSet() && banedParents.isEmpty() == false) { String reason = banedParents.get(task.getParentTaskId()); if (reason != null) { try { @@ -241,7 +241,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen CancellableTaskHolder holder = taskEntry.getValue(); CancellableTask task = holder.getTask(); TaskId parentTaskId = task.getParentTaskId(); - if (parentTaskId.isSet() == false && lastDiscoveryNodes.nodeExists(parentTaskId.getNodeId()) == false) { + if (parentTaskId.isSet() && lastDiscoveryNodes.nodeExists(parentTaskId.getNodeId()) == false) { if (task.cancelOnParentLeaving()) { holder.cancel("Coordinating node [" + parentTaskId.getNodeId() + "] left the cluster"); } diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java index b22d93ef6b2..90dbfaa8c60 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java @@ -118,9 +118,9 @@ public class TasksIT extends ESIntegTestCase { // Verify that one of these tasks is a parent of another task if (tasks.get(0).getParentTaskId().isSet()) { - assertParentTask(Collections.singletonList(tasks.get(1)), tasks.get(0)); - } else { assertParentTask(Collections.singletonList(tasks.get(0)), tasks.get(1)); + } else { + assertParentTask(Collections.singletonList(tasks.get(1)), tasks.get(0)); } } @@ -474,7 +474,7 @@ public class TasksIT extends ESIntegTestCase { */ private void assertParentTask(List tasks, TaskInfo parentTask) { for (TaskInfo task : tasks) { - assertFalse(task.getParentTaskId().isSet()); + assertTrue(task.getParentTaskId().isSet()); assertEquals(parentTask.getNode().getId(), task.getParentTaskId().getNodeId()); assertTrue(Strings.hasLength(task.getParentTaskId().getNodeId())); assertEquals(parentTask.getId(), task.getParentTaskId().getId()); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index 64d69a4864f..4b478b52bd0 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -604,7 +604,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase { @Override protected TestTaskResponse taskOperation(TestTasksRequest request, Task task) { logger.info("Task action on node {}", node); - if (failTaskOnNode == node && task.getParentTaskId().isSet() == false) { + if (failTaskOnNode == node && task.getParentTaskId().isSet()) { logger.info("Failing on node {}", node); throw new RuntimeException("Task level failure"); } From 8202bf212cddafcff79ca78d00efdee76e6d6ff4 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 21 Mar 2016 16:02:08 -0400 Subject: [PATCH 37/45] Don't wait for completion of list tasks tasks when wait_for_completion flag is set Waiting for completion of list tasks tasks can cause an infinite loop of a list tasks task waiting for its own completion or completion of its children. To reproduce run: ``` curl "localhost:9200/_tasks?wait_for_completion" ``` --- .../node/tasks/list/TransportListTasksAction.java | 8 +++++++- .../cluster/node/tasks/RestListTasksAction.java | 3 +++ .../action/admin/cluster/node/tasks/TasksIT.java | 13 +++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java index d5175e263e7..e6ea002a794 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java @@ -84,7 +84,13 @@ public class TransportListTasksAction extends TransportTasksAction { while (System.nanoTime() - timeoutTime < 0) { - if (taskManager.getTask(t.getId()) == null) { + Task task = taskManager.getTask(t.getId()); + if (task == null) { + return; + } + if (task.getAction().startsWith(ListTasksAction.NAME)) { + // It doesn't make sense to wait for List Tasks and it can cause an infinite loop of the task waiting + // for itself of one of its child tasks return; } try { diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java index 9a9d1991298..168d091968e 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestController; @@ -51,6 +52,7 @@ public class RestListTasksAction extends BaseRestHandler { String[] actions = Strings.splitStringByCommaToArray(request.param("actions")); TaskId parentTaskId = new TaskId(request.param("parent_task_id")); boolean waitForCompletion = request.paramAsBoolean("wait_for_completion", false); + TimeValue timeout = request.paramAsTime("timeout", null); ListTasksRequest listTasksRequest = new ListTasksRequest(); listTasksRequest.setTaskId(taskId); @@ -59,6 +61,7 @@ public class RestListTasksAction extends BaseRestHandler { listTasksRequest.setActions(actions); listTasksRequest.setParentTaskId(parentTaskId); listTasksRequest.setWaitForCompletion(waitForCompletion); + listTasksRequest.setTimeout(timeout); client.admin().cluster().listTasks(listTasksRequest, new RestToXContentListener<>(channel)); } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java index 90dbfaa8c60..e9ade3ef5af 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java @@ -21,6 +21,7 @@ package org.elasticsearch.action.admin.cluster.node.tasks; import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.TaskOperationFailure; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; @@ -59,6 +60,7 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; +import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.emptyCollectionOf; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -406,6 +408,17 @@ public class TasksIT extends ESIntegTestCase { assertThat(waitResponseFuture.get().getTasks(), emptyCollectionOf(TaskInfo.class)); } + public void testTasksWaitForAllTask() throws Exception { + // Spin up a request to wait for all tasks in the cluster to make sure it doesn't cause an infinite loop + ListTasksResponse response = client().admin().cluster().prepareListTasks().setWaitForCompletion(true) + .setTimeout(timeValueSeconds(10)).get(); + + // It should finish quickly and without complaint and list the list tasks themselves + assertThat(response.getNodeFailures(), emptyCollectionOf(FailedNodeException.class)); + assertThat(response.getTaskFailures(), emptyCollectionOf(TaskOperationFailure.class)); + assertThat(response.getTasks().size(), greaterThanOrEqualTo(1)); + } + @Override public void tearDown() throws Exception { for (Map.Entry, RecordingTaskManagerListener> entry : listeners.entrySet()) { From 7ecfa6e2adc472cd897aeb2c245f3a095fc65d78 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 23 Mar 2016 23:11:44 -0400 Subject: [PATCH 38/45] Add debug logging for Vagrant upgrade test --- .../src/test/resources/packaging/scripts/80_upgrade.bats | 2 ++ .../resources/packaging/scripts/packaging_test_utils.bash | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats b/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats index 48bf2aca4e3..deeb37d54e5 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats +++ b/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats @@ -42,6 +42,7 @@ setup() { @test "[UPGRADE] install old version" { clean_before_test install_package -v $(cat upgrade_from_version) + perl -p -i -e 's/es.logger.level: INFO/es.logger.level: DEBUG/' /etc/elasticsearch/logging.yml } @test "[UPGRADE] start old version" { @@ -80,6 +81,7 @@ setup() { @test "[UPGRADE] install version under test" { install_package -u + perl -p -i -e 's/es.logger.level: INFO/es.logger.level: DEBUG/' /etc/elasticsearch/logging.yml } @test "[UPGRADE] start version under test" { diff --git a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash index 3d449ebf901..fbb86b41c87 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash +++ b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash @@ -385,6 +385,10 @@ wait_for_elasticsearch_status() { local desiredStatus=${1:-green} local index=$2 + if [ -f /var/log/elasitcsearch/elasticsearch.log ]; then + cat /var/log/elasticsearch/elasticsearch.log >&3 + fi + echo "Making sure elasticsearch is up..." wget -O - --retry-connrefused --waitretry=1 --timeout=60 --tries 60 http://localhost:9200 || { echo "Looks like elasticsearch never started. Here is its log:" @@ -394,6 +398,7 @@ wait_for_elasticsearch_status() { echo "The elasticsearch log doesn't exist. Maybe /var/log/messages has something:" tail -n20 /var/log/messages fi + cat /var/log/elasticsearch/elasticsearch.log >&3 false } @@ -408,6 +413,7 @@ wait_for_elasticsearch_status() { echo "Connected" else echo "Unable to connect to Elastisearch" + cat /var/log/elasticsearch/elasticsearch.log >&3 false fi From b2573858b60c60e06853914129336e98aa146651 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Thu, 24 Mar 2016 08:36:08 +0100 Subject: [PATCH 39/45] Version: Set version to 5.0.0-alpha1 Change version, required a minor fix in the RPM building. In case of a alpha/beta version, the release will contain alpha/beta as the RPM version cannot contains dashes/tildes. --- buildSrc/version.properties | 2 +- .../main/java/org/elasticsearch/Version.java | 10 +++---- .../admin/indices/stats/CommonStatsFlags.java | 4 +-- .../cluster/routing/ShardRouting.java | 2 +- .../gateway/PrimaryShardAllocator.java | 4 +-- .../index/analysis/AnalysisService.java | 2 +- .../index/engine/SegmentsStats.java | 4 +-- .../index/mapper/FieldMapper.java | 6 ++--- .../index/mapper/MappedFieldType.java | 2 +- .../index/mapper/core/ByteFieldMapper.java | 4 +-- .../index/mapper/core/DateFieldMapper.java | 4 +-- .../index/mapper/core/DoubleFieldMapper.java | 4 +-- .../index/mapper/core/FloatFieldMapper.java | 4 +-- .../index/mapper/core/IntegerFieldMapper.java | 4 +-- .../index/mapper/core/LongFieldMapper.java | 4 +-- .../index/mapper/core/ShortFieldMapper.java | 4 +-- .../index/mapper/core/StringFieldMapper.java | 8 +++--- .../index/mapper/core/TypeParsers.java | 4 +-- .../index/mapper/geo/GeoShapeFieldMapper.java | 3 +-- .../mapper/internal/SourceFieldMapper.java | 2 +- .../index/mapper/ip/IpFieldMapper.java | 2 +- .../index/mapper/object/DynamicTemplate.java | 2 +- .../percolator/PercolatorQueryCache.java | 5 +--- .../index/query/PercolatorQueryBuilder.java | 2 +- .../index/similarity/SimilarityService.java | 4 +-- .../java/org/elasticsearch/VersionTests.java | 26 ++++++++++--------- .../discovery/zen/ZenDiscoveryIT.java | 8 +++--- .../index/analysis/AnalysisServiceTests.java | 6 ++--- .../index/mapper/DynamicTemplateTests.java | 10 +++---- distribution/rpm/build.gradle | 9 ++++++- qa/backwards-5.0/build.gradle | 2 +- .../test/test/VersionUtilsTests.java | 12 ++++----- 32 files changed, 87 insertions(+), 82 deletions(-) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 39c32192052..13c0885a61a 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,4 +1,4 @@ -elasticsearch = 5.0.0 +elasticsearch = 5.0.0-alpha1 lucene = 6.0.0-snapshot-f0aa4fc # optional dependencies diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java index 8b65adf170d..fe794d96661 100644 --- a/core/src/main/java/org/elasticsearch/Version.java +++ b/core/src/main/java/org/elasticsearch/Version.java @@ -64,9 +64,9 @@ public class Version { public static final Version V_2_2_1 = new Version(V_2_2_1_ID, org.apache.lucene.util.Version.LUCENE_5_4_1); public static final int V_2_3_0_ID = 2030099; public static final Version V_2_3_0 = new Version(V_2_3_0_ID, org.apache.lucene.util.Version.LUCENE_5_5_0); - public static final int V_5_0_0_ID = 5000099; - public static final Version V_5_0_0 = new Version(V_5_0_0_ID, org.apache.lucene.util.Version.LUCENE_6_0_0); - public static final Version CURRENT = V_5_0_0; + public static final int V_5_0_0_alpha1_ID = 5000001; + public static final Version V_5_0_0_alpha1 = new Version(V_5_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_6_0_0); + public static final Version CURRENT = V_5_0_0_alpha1; static { assert CURRENT.luceneVersion.equals(org.apache.lucene.util.Version.LATEST) : "Version must be upgraded to [" @@ -79,8 +79,8 @@ public class Version { public static Version fromId(int id) { switch (id) { - case V_5_0_0_ID: - return V_5_0_0; + case V_5_0_0_alpha1_ID: + return V_5_0_0_alpha1; case V_2_3_0_ID: return V_2_3_0; case V_2_2_1_ID: diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java index f00f22da536..e2f250dd577 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java @@ -190,7 +190,7 @@ public class CommonStatsFlags implements Streamable, Cloneable { out.writeStringArrayNullable(groups); out.writeStringArrayNullable(fieldDataFields); out.writeStringArrayNullable(completionDataFields); - if (out.getVersion().onOrAfter(Version.V_5_0_0)) { + if (out.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) { out.writeBoolean(includeSegmentFileSizes); } } @@ -208,7 +208,7 @@ public class CommonStatsFlags implements Streamable, Cloneable { groups = in.readStringArray(); fieldDataFields = in.readStringArray(); completionDataFields = in.readStringArray(); - if (in.getVersion().onOrAfter(Version.V_5_0_0)) { + if (in.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) { includeSegmentFileSizes = in.readBoolean(); } else { includeSegmentFileSizes = false; diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java index a5975deb9cf..269a238e710 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java @@ -266,7 +266,7 @@ public final class ShardRouting implements Streamable, ToXContent { return false; } - if (indexMetaData.activeAllocationIds(id()).isEmpty() && indexMetaData.getCreationVersion().onOrAfter(Version.V_5_0_0)) { + if (indexMetaData.activeAllocationIds(id()).isEmpty() && indexMetaData.getCreationVersion().onOrAfter(Version.V_5_0_0_alpha1)) { // when no shards with this id have ever been active for this index return false; } diff --git a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java index a456da0779d..e7447e301c4 100644 --- a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java +++ b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java @@ -118,7 +118,7 @@ public abstract class PrimaryShardAllocator extends AbstractComponent { final boolean enoughAllocationsFound; if (lastActiveAllocationIds.isEmpty()) { - assert Version.indexCreated(indexMetaData.getSettings()).before(Version.V_5_0_0) : "trying to allocated a primary with an empty allocation id set, but index is new"; + assert Version.indexCreated(indexMetaData.getSettings()).before(Version.V_5_0_0_alpha1) : "trying to allocated a primary with an empty allocation id set, but index is new"; // when we load an old index (after upgrading cluster) or restore a snapshot of an old index // fall back to old version-based allocation mode // Note that once the shard has been active, lastActiveAllocationIds will be non-empty @@ -128,7 +128,7 @@ public abstract class PrimaryShardAllocator extends AbstractComponent { } else { enoughAllocationsFound = isEnoughVersionBasedAllocationsFound(indexMetaData, nodeShardsResult); } - logger.debug("[{}][{}]: version-based allocation for pre-{} index found {} allocations of {}", shard.index(), shard.id(), Version.V_5_0_0, nodeShardsResult.allocationsFound, shard); + logger.debug("[{}][{}]: version-based allocation for pre-{} index found {} allocations of {}", shard.index(), shard.id(), Version.V_5_0_0_alpha1, nodeShardsResult.allocationsFound, shard); } else { assert lastActiveAllocationIds.isEmpty() == false; // use allocation ids to select nodes diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java index 453552b9dd1..b9146df8c96 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java @@ -127,7 +127,7 @@ public class AnalysisService extends AbstractIndexComponent implements Closeable } if (analyzers.containsKey("default_index")) { final Version createdVersion = indexSettings.getIndexVersionCreated(); - if (createdVersion.onOrAfter(Version.V_5_0_0)) { + if (createdVersion.onOrAfter(Version.V_5_0_0_alpha1)) { throw new IllegalArgumentException("setting [index.analysis.analyzer.default_index] is not supported anymore, use [index.analysis.analyzer.default] instead for index [" + index().getName() + "]"); } else { deprecationLogger.deprecated("setting [index.analysis.analyzer.default_index] is deprecated, use [index.analysis.analyzer.default] instead for index [{}]", index().getName()); diff --git a/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java b/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java index d2c4a3c14c0..10948f35d54 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java +++ b/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java @@ -348,7 +348,7 @@ public class SegmentsStats implements Streamable, ToXContent { indexWriterMaxMemoryInBytes = in.readLong(); bitsetMemoryInBytes = in.readLong(); - if (in.getVersion().onOrAfter(Version.V_5_0_0)) { + if (in.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) { int size = in.readVInt(); ImmutableOpenMap.Builder map = ImmutableOpenMap.builder(size); for (int i = 0; i < size; i++) { @@ -376,7 +376,7 @@ public class SegmentsStats implements Streamable, ToXContent { out.writeLong(indexWriterMaxMemoryInBytes); out.writeLong(bitsetMemoryInBytes); - if (out.getVersion().onOrAfter(Version.V_5_0_0)) { + if (out.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) { out.writeVInt(fileSizes.size()); for (Iterator> it = fileSizes.iterator(); it.hasNext();) { ObjectObjectCursor entry = it.next(); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 9f6d0ea270a..8077a732142 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -219,7 +219,7 @@ public abstract class FieldMapper extends Mapper implements Cloneable { } protected boolean defaultDocValues(Version indexCreated) { - if (indexCreated.onOrAfter(Version.V_5_0_0)) { + if (indexCreated.onOrAfter(Version.V_5_0_0_alpha1)) { // add doc values by default to keyword (boolean, numerics, etc.) fields return fieldType.tokenized() == false; } else { @@ -229,7 +229,7 @@ public abstract class FieldMapper extends Mapper implements Cloneable { protected void setupFieldType(BuilderContext context) { fieldType.setName(buildFullName(context)); - if (context.indexCreatedVersion().before(Version.V_5_0_0)) { + if (context.indexCreatedVersion().before(Version.V_5_0_0_alpha1)) { fieldType.setOmitNorms(fieldType.omitNorms() && fieldType.boost() == 1.0f); } if (fieldType.indexAnalyzer() == null && fieldType.tokenized() == false && fieldType.indexOptions() != IndexOptions.NONE) { @@ -289,7 +289,7 @@ public abstract class FieldMapper extends Mapper implements Cloneable { if (!customBoost() // don't set boosts eg. on dv fields && field.fieldType().indexOptions() != IndexOptions.NONE - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(fieldType().boost()); } context.doc().add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 054f307d3dd..9eafa8ccd71 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -341,7 +341,7 @@ public abstract class MappedFieldType extends FieldType { public Query termQuery(Object value, @Nullable QueryShardContext context) { TermQuery query = new TermQuery(createTerm(value)); if (boost == 1f || - (context != null && context.indexVersionCreated().before(Version.V_5_0_0))) { + (context != null && context.indexVersionCreated().before(Version.V_5_0_0_alpha1))) { return query; } return new BoostQuery(query, boost); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java index 2c7555c7fe6..c42f0b7cd42 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java @@ -261,7 +261,7 @@ public class ByteFieldMapper extends NumberFieldMapper { context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { XContentParser.Token token; String currentFieldName = null; Byte objValue = fieldType().nullValue(); @@ -294,7 +294,7 @@ public class ByteFieldMapper extends NumberFieldMapper { } if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomByteNumericField field = new CustomByteNumericField(value, fieldType()); - if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(boost); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index 7c6381b7c1c..5d66fb62c76 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -488,7 +488,7 @@ public class DateFieldMapper extends NumberFieldMapper { } else if (token == XContentParser.Token.VALUE_NUMBER) { dateAsString = parser.text(); } else if (token == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { String currentFieldName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -523,7 +523,7 @@ public class DateFieldMapper extends NumberFieldMapper { if (value != null) { if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomLongNumericField field = new CustomLongNumericField(value, fieldType()); - if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(boost); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java index 02bdfdf1dd1..69d56439b33 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java @@ -253,7 +253,7 @@ public class DoubleFieldMapper extends NumberFieldMapper { context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { XContentParser.Token token; String currentFieldName = null; Double objValue = fieldType().nullValue(); @@ -287,7 +287,7 @@ public class DoubleFieldMapper extends NumberFieldMapper { if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomDoubleNumericField field = new CustomDoubleNumericField(value, fieldType()); - if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(boost); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java index 4182b37b08f..48b9ca95875 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java @@ -265,7 +265,7 @@ public class FloatFieldMapper extends NumberFieldMapper { context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { XContentParser.Token token; String currentFieldName = null; Float objValue = fieldType().nullValue(); @@ -299,7 +299,7 @@ public class FloatFieldMapper extends NumberFieldMapper { if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomFloatNumericField field = new CustomFloatNumericField(value, fieldType()); - if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(boost); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java index 3e1e7a538a2..860377d0879 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java @@ -270,7 +270,7 @@ public class IntegerFieldMapper extends NumberFieldMapper { context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { XContentParser.Token token; String currentFieldName = null; Integer objValue = fieldType().nullValue(); @@ -307,7 +307,7 @@ public class IntegerFieldMapper extends NumberFieldMapper { protected void addIntegerFields(ParseContext context, List fields, int value, float boost) { if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomIntegerNumericField field = new CustomIntegerNumericField(value, fieldType()); - if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(boost); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java index 8c346c2e0c1..b261df7a221 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java @@ -258,7 +258,7 @@ public class LongFieldMapper extends NumberFieldMapper { context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { XContentParser.Token token; String currentFieldName = null; Long objValue = fieldType().nullValue(); @@ -291,7 +291,7 @@ public class LongFieldMapper extends NumberFieldMapper { } if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomLongNumericField field = new CustomLongNumericField(value, fieldType()); - if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(boost); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java index 2f5bd3d974c..eb00e0744c8 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java @@ -266,7 +266,7 @@ public class ShortFieldMapper extends NumberFieldMapper { context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { XContentParser.Token token; String currentFieldName = null; Short objValue = fieldType().nullValue(); @@ -299,7 +299,7 @@ public class ShortFieldMapper extends NumberFieldMapper { } if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomShortNumericField field = new CustomShortNumericField(value, fieldType()); - if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(boost); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java index f58db38a3dd..d19c6e31b0a 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java @@ -193,7 +193,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc @Override public Mapper.Builder parse(String fieldName, Map node, ParserContext parserContext) throws MapperParsingException { - if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0)) { + if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha1)) { // Automatically upgrade simple mappings for ease of upgrade, otherwise fail if (SUPPORTED_PARAMETERS_FOR_AUTO_UPGRADE.containsAll(node.keySet())) { deprecationLogger.deprecated("The [string] field is deprecated, please use [text] or [keyword] instead on [{}]", @@ -490,7 +490,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc int positionIncrementGap, int ignoreAbove, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); - if (Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0)) { + if (Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0_alpha1)) { throw new IllegalArgumentException("The [string] type is removed in 5.0. You should now use either a [text] " + "or [keyword] field instead for field [" + fieldType.name() + "]"); } @@ -573,7 +573,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { Field field = new Field(fieldType().name(), valueAndBoost.value(), fieldType()); - if (valueAndBoost.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (valueAndBoost.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(valueAndBoost.boost()); } fields.add(field); @@ -600,7 +600,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc return new ValueAndBoost(nullValue, defaultBoost); } if (parser.currentToken() == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { XContentParser.Token token; String currentFieldName = null; String value = nullValue; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java index 47471254e13..7fef5354e1a 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java @@ -250,7 +250,7 @@ public class TypeParsers { } else if (propName.equals("boost")) { builder.boost(nodeFloatValue(propNode)); iterator.remove(); - } else if (parserContext.indexVersionCreated().before(Version.V_5_0_0) + } else if (parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1) && parseNorms(builder, propName, propNode, parserContext)) { iterator.remove(); } else if (propName.equals("index_options")) { @@ -434,7 +434,7 @@ public class TypeParsers { } private static SimilarityProvider resolveSimilarity(Mapper.TypeParser.ParserContext parserContext, String name, String value) { - if (parserContext.indexVersionCreated().before(Version.V_5_0_0) && "default".equals(value)) { + if (parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1) && "default".equals(value)) { // "default" similarity has been renamed into "classic" in 3.x. value = SimilarityService.DEFAULT_SIMILARITY; } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java index 57778fa8d25..1a0f2561a40 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java @@ -46,7 +46,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.core.DoubleFieldMapper; import java.io.IOException; import java.util.Iterator; @@ -454,7 +453,7 @@ public class GeoShapeFieldMapper extends FieldMapper { } for (Field field : fields) { if (!customBoost() && - fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(fieldType().boost()); } context.doc().add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java index a102f03a5f1..3eeca0352a7 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java @@ -124,7 +124,7 @@ public class SourceFieldMapper extends MetadataFieldMapper { if (fieldName.equals("enabled")) { builder.enabled(lenientNodeBooleanValue(fieldNode)); iterator.remove(); - } else if ("format".equals(fieldName) && parserContext.indexVersionCreated().before(Version.V_5_0_0)) { + } else if ("format".equals(fieldName) && parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1)) { // ignore on old indices, reject on and after 5.0 iterator.remove(); } else if (fieldName.equals("includes")) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java index d0ba6aaf3ab..b754c163669 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java @@ -322,7 +322,7 @@ public class IpFieldMapper extends NumberFieldMapper { final long value = ipToLong(ipAsString); if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomLongNumericField field = new CustomLongNumericField(value, fieldType()); - if (fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(fieldType().boost()); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java b/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java index eaaa1ddfc69..3deb487e50b 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java @@ -100,7 +100,7 @@ public class DynamicTemplate implements ToXContent { matchPattern = entry.getValue().toString(); } else if ("mapping".equals(propName)) { mapping = (Map) entry.getValue(); - } else if (indexVersionCreated.onOrAfter(Version.V_5_0_0)) { + } else if (indexVersionCreated.onOrAfter(Version.V_5_0_0_alpha1)) { // unknown parameters were ignored before but still carried through serialization // so we need to ignore them at parsing time for old indices throw new IllegalArgumentException("Illegal dynamic template parameter: [" + propName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java index 459ab1df0b8..78d8af06827 100644 --- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java +++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java @@ -29,7 +29,6 @@ import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Query; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; @@ -37,11 +36,9 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.cache.Cache; import org.elasticsearch.common.cache.CacheBuilder; -import org.elasticsearch.common.lucene.index.ElasticsearchLeafReader; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContent; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; @@ -143,7 +140,7 @@ public final class PercolatorQueryCache extends AbstractIndexComponent } IntObjectHashMap queries = new IntObjectHashMap<>(); - boolean legacyLoading = indexVersionCreated.before(Version.V_5_0_0); + boolean legacyLoading = indexVersionCreated.before(Version.V_5_0_0_alpha1); PostingsEnum postings = leafReader.postings(new Term(TypeFieldMapper.NAME, PercolatorFieldMapper.TYPE_NAME), PostingsEnum.NONE); if (postings != null) { if (legacyLoading) { diff --git a/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryBuilder.java index 5cb1e54d203..7ca70eef6aa 100644 --- a/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryBuilder.java @@ -301,7 +301,7 @@ public class PercolatorQueryBuilder extends AbstractQueryBuilder entry : similaritySettings.entrySet()) { String name = entry.getKey(); // Starting with v5.0 indices, it should no longer be possible to redefine built-in similarities - if(BUILT_IN.containsKey(name) && indexSettings.getIndexVersionCreated().onOrAfter(Version.V_5_0_0)) { + if(BUILT_IN.containsKey(name) && indexSettings.getIndexVersionCreated().onOrAfter(Version.V_5_0_0_alpha1)) { throw new IllegalArgumentException("Cannot redefine built-in Similarity [" + name + "]"); } Settings settings = entry.getValue(); @@ -83,7 +83,7 @@ public final class SimilarityService extends AbstractIndexComponent { } for (Map.Entry entry : addSimilarities(similaritySettings, DEFAULTS).entrySet()) { // Avoid overwriting custom providers for indices older that v5.0 - if (providers.containsKey(entry.getKey()) && indexSettings.getIndexVersionCreated().before(Version.V_5_0_0)) { + if (providers.containsKey(entry.getKey()) && indexSettings.getIndexVersionCreated().before(Version.V_5_0_0_alpha1)) { continue; } providers.put(entry.getKey(), entry.getValue()); diff --git a/core/src/test/java/org/elasticsearch/VersionTests.java b/core/src/test/java/org/elasticsearch/VersionTests.java index 17a41c30275..65c91f5daab 100644 --- a/core/src/test/java/org/elasticsearch/VersionTests.java +++ b/core/src/test/java/org/elasticsearch/VersionTests.java @@ -32,7 +32,7 @@ import java.util.Locale; import java.util.Map; import static org.elasticsearch.Version.V_2_2_0; -import static org.elasticsearch.Version.V_5_0_0; +import static org.elasticsearch.Version.V_5_0_0_alpha1; import static org.elasticsearch.test.VersionUtils.randomVersion; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.containsString; @@ -42,21 +42,21 @@ import static org.hamcrest.Matchers.sameInstance; public class VersionTests extends ESTestCase { public void testVersionComparison() throws Exception { - assertThat(V_2_2_0.before(V_5_0_0), is(true)); + assertThat(V_2_2_0.before(V_5_0_0_alpha1), is(true)); assertThat(V_2_2_0.before(V_2_2_0), is(false)); - assertThat(V_5_0_0.before(V_2_2_0), is(false)); + assertThat(V_5_0_0_alpha1.before(V_2_2_0), is(false)); - assertThat(V_2_2_0.onOrBefore(V_5_0_0), is(true)); + assertThat(V_2_2_0.onOrBefore(V_5_0_0_alpha1), is(true)); assertThat(V_2_2_0.onOrBefore(V_2_2_0), is(true)); - assertThat(V_5_0_0.onOrBefore(V_2_2_0), is(false)); + assertThat(V_5_0_0_alpha1.onOrBefore(V_2_2_0), is(false)); - assertThat(V_2_2_0.after(V_5_0_0), is(false)); + assertThat(V_2_2_0.after(V_5_0_0_alpha1), is(false)); assertThat(V_2_2_0.after(V_2_2_0), is(false)); - assertThat(V_5_0_0.after(V_2_2_0), is(true)); + assertThat(V_5_0_0_alpha1.after(V_2_2_0), is(true)); - assertThat(V_2_2_0.onOrAfter(V_5_0_0), is(false)); + assertThat(V_2_2_0.onOrAfter(V_5_0_0_alpha1), is(false)); assertThat(V_2_2_0.onOrAfter(V_2_2_0), is(true)); - assertThat(V_5_0_0.onOrAfter(V_2_2_0), is(true)); + assertThat(V_5_0_0_alpha1.onOrAfter(V_2_2_0), is(true)); assertTrue(Version.fromString("5.0.0-alpha2").onOrAfter(Version.fromString("5.0.0-alpha1"))); assertTrue(Version.fromString("5.0.0").onOrAfter(Version.fromString("5.0.0-beta2"))); @@ -133,7 +133,7 @@ public class VersionTests extends ESTestCase { public void testIndexCreatedVersion() { // an actual index has a IndexMetaData.SETTING_INDEX_UUID - final Version version = randomFrom(Version.V_2_0_0, Version.V_2_3_0, Version.V_5_0_0); + final Version version = randomFrom(Version.V_2_0_0, Version.V_2_3_0, Version.V_5_0_0_alpha1); assertEquals(version, Version.indexCreated(Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, "foo").put(IndexMetaData.SETTING_VERSION_CREATED, version).build())); } @@ -142,13 +142,13 @@ public class VersionTests extends ESTestCase { assertThat(Version.V_2_1_0.minimumCompatibilityVersion(), equalTo(Version.V_2_0_0)); assertThat(Version.V_2_2_0.minimumCompatibilityVersion(), equalTo(Version.V_2_0_0)); assertThat(Version.V_2_3_0.minimumCompatibilityVersion(), equalTo(Version.V_2_0_0)); - assertThat(Version.V_5_0_0.minimumCompatibilityVersion(), equalTo(Version.V_5_0_0)); + assertThat(Version.V_5_0_0_alpha1.minimumCompatibilityVersion(), equalTo(Version.V_5_0_0_alpha1)); } public void testToString() { // with 2.0.beta we lowercase assertEquals("2.0.0-beta1", Version.V_2_0_0_beta1.toString()); - assertEquals("5.0.0", Version.V_5_0_0.toString()); + assertEquals("5.0.0-alpha1", Version.V_5_0_0_alpha1.toString()); assertEquals("2.3.0", Version.V_2_3_0.toString()); assertEquals("0.90.0.Beta1", Version.fromString("0.90.0.Beta1").toString()); assertEquals("1.0.0.Beta1", Version.fromString("1.0.0.Beta1").toString()); @@ -240,6 +240,8 @@ public class VersionTests extends ESTestCase { number = number.replace("-beta", "_beta"); } else if (v.isRC()) { number = number.replace("-rc", "_rc"); + } else if (v.isAlpha()) { + number = number.replace("-alpha", "_alpha"); } assertEquals("V_" + number.replace('.', '_'), constantName); } else { diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java index b7ce4c305e1..d14bd583c44 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java @@ -275,7 +275,7 @@ public class ZenDiscoveryIT extends ESIntegTestCase { Settings nodeSettings = Settings.settingsBuilder() .put("discovery.type", "zen") // <-- To override the local setting if set externally .build(); - String nodeName = internalCluster().startNode(nodeSettings, Version.V_5_0_0); + String nodeName = internalCluster().startNode(nodeSettings, Version.V_5_0_0_alpha1); ZenDiscovery zenDiscovery = (ZenDiscovery) internalCluster().getInstance(Discovery.class, nodeName); ClusterService clusterService = internalCluster().getInstance(ClusterService.class, nodeName); DiscoveryNode node = new DiscoveryNode("_node_id", new InetSocketTransportAddress(InetAddress.getByName("0.0.0.0"), 0), Version.V_2_0_0); @@ -292,13 +292,13 @@ public class ZenDiscoveryIT extends ESIntegTestCase { }); assertThat(holder.get(), notNullValue()); - assertThat(holder.get().getMessage(), equalTo("Can't handle join request from a node with a version [2.0.0] that is lower than the minimum compatible version [" + Version.V_5_0_0.minimumCompatibilityVersion() + "]")); + assertThat(holder.get().getMessage(), equalTo("Can't handle join request from a node with a version [2.0.0] that is lower than the minimum compatible version [" + Version.V_5_0_0_alpha1.minimumCompatibilityVersion() + "]")); } public void testJoinElectedMaster_incompatibleMinVersion() { - ElectMasterService electMasterService = new ElectMasterService(Settings.EMPTY, Version.V_5_0_0); + ElectMasterService electMasterService = new ElectMasterService(Settings.EMPTY, Version.V_5_0_0_alpha1); - DiscoveryNode node = new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.V_5_0_0); + DiscoveryNode node = new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.V_5_0_0_alpha1); assertThat(electMasterService.electMaster(Collections.singletonList(node)), sameInstance(node)); node = new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.V_2_0_0); assertThat("Can't join master because version 2.0.0 is lower than the minimum compatable version 5.0.0 can support", electMasterService.electMaster(Collections.singletonList(node)), nullValue()); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java index 1fd9655e8de..f665de6216d 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java @@ -76,7 +76,7 @@ public class AnalysisServiceTests extends ESTestCase { } public void testOverrideDefaultIndexAnalyzer() { - Version version = VersionUtils.randomVersionBetween(getRandom(), Version.V_5_0_0, Version.CURRENT); + Version version = VersionUtils.randomVersionBetween(getRandom(), Version.V_5_0_0_alpha1, Version.CURRENT); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); try { AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings), @@ -90,7 +90,7 @@ public class AnalysisServiceTests extends ESTestCase { } public void testBackCompatOverrideDefaultIndexAnalyzer() { - Version version = VersionUtils.randomVersionBetween(getRandom(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0)); + Version version = VersionUtils.randomVersionBetween(getRandom(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1)); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings), Collections.singletonMap("default_index", analyzerProvider("default_index")), @@ -112,7 +112,7 @@ public class AnalysisServiceTests extends ESTestCase { } public void testBackCompatOverrideDefaultIndexAndSearchAnalyzer() { - Version version = VersionUtils.randomVersionBetween(getRandom(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0)); + Version version = VersionUtils.randomVersionBetween(getRandom(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1)); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); Map analyzers = new HashMap<>(); analyzers.put("default_index", analyzerProvider("default_index")); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java index 8ee8332b708..31b65e84aa0 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java @@ -39,7 +39,7 @@ public class DynamicTemplateTests extends ESTestCase { templateDef.put("random_param", "random_value"); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0)); + () -> DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha1)); assertEquals("Illegal dynamic template parameter: [random_param]", e.getMessage()); // but no issues on 2.x for bw compat @@ -54,7 +54,7 @@ public class DynamicTemplateTests extends ESTestCase { Map templateDef = new HashMap<>(); templateDef.put("match_mapping_type", "string"); templateDef.put("mapping", Collections.singletonMap("store", true)); - DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha1); XContentBuilder builder = JsonXContent.contentBuilder(); template.toXContent(builder, ToXContent.EMPTY_PARAMS); assertEquals("{\"match_mapping_type\":\"string\",\"mapping\":{\"store\":true}}", builder.string()); @@ -64,7 +64,7 @@ public class DynamicTemplateTests extends ESTestCase { templateDef.put("match", "*name"); templateDef.put("unmatch", "first_name"); templateDef.put("mapping", Collections.singletonMap("store", true)); - template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0); + template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha1); builder = JsonXContent.contentBuilder(); template.toXContent(builder, ToXContent.EMPTY_PARAMS); assertEquals("{\"match\":\"*name\",\"unmatch\":\"first_name\",\"mapping\":{\"store\":true}}", builder.string()); @@ -74,7 +74,7 @@ public class DynamicTemplateTests extends ESTestCase { templateDef.put("path_match", "*name"); templateDef.put("path_unmatch", "first_name"); templateDef.put("mapping", Collections.singletonMap("store", true)); - template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0); + template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha1); builder = JsonXContent.contentBuilder(); template.toXContent(builder, ToXContent.EMPTY_PARAMS); assertEquals("{\"path_match\":\"*name\",\"path_unmatch\":\"first_name\",\"mapping\":{\"store\":true}}", @@ -85,7 +85,7 @@ public class DynamicTemplateTests extends ESTestCase { templateDef.put("match", "^a$"); templateDef.put("match_pattern", "regex"); templateDef.put("mapping", Collections.singletonMap("store", true)); - template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0); + template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha1); builder = JsonXContent.contentBuilder(); template.toXContent(builder, ToXContent.EMPTY_PARAMS); assertEquals("{\"match\":\"^a$\",\"match_pattern\":\"regex\",\"mapping\":{\"store\":true}}", builder.string()); diff --git a/distribution/rpm/build.gradle b/distribution/rpm/build.gradle index 2ab78fe7e41..177ab518a10 100644 --- a/distribution/rpm/build.gradle +++ b/distribution/rpm/build.gradle @@ -25,7 +25,14 @@ task buildRpm(type: Rpm) { packageGroup 'Application/Internet' prefix '/usr' packager 'Elasticsearch' - release '1' + if (version.contains('~')) { + def tokenized = version.tokenize('~') + version tokenized[0] + release tokenized[1] + } else { + version version + release '1' + } arch NOARCH os LINUX // TODO ospackage doesn't support icon but we used to have one diff --git a/qa/backwards-5.0/build.gradle b/qa/backwards-5.0/build.gradle index 93d361c989c..c86b21dfea4 100644 --- a/qa/backwards-5.0/build.gradle +++ b/qa/backwards-5.0/build.gradle @@ -18,6 +18,6 @@ integTest { cluster { numNodes = 2 numBwcNodes = 1 - bwcVersion = "5.0.0-SNAPSHOT" // this is the same as the current version until we released the first RC + bwcVersion = "5.0.0-alpha1-SNAPSHOT" // this is the same as the current version until we released the first RC } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java index ea1929a55b0..97683b620f0 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java @@ -46,14 +46,14 @@ public class VersionUtilsTests extends ESTestCase { assertTrue(got.onOrBefore(Version.CURRENT)); // sub range - got = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0); + got = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha1); assertTrue(got.onOrAfter(Version.V_2_0_0)); - assertTrue(got.onOrBefore(Version.V_5_0_0)); + assertTrue(got.onOrBefore(Version.V_5_0_0_alpha1)); // unbounded lower - got = VersionUtils.randomVersionBetween(random(), null, Version.V_5_0_0); + got = VersionUtils.randomVersionBetween(random(), null, Version.V_5_0_0_alpha1); assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); - assertTrue(got.onOrBefore(Version.V_5_0_0)); + assertTrue(got.onOrBefore(Version.V_5_0_0_alpha1)); got = VersionUtils.randomVersionBetween(random(), null, VersionUtils.allVersions().get(0)); assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); assertTrue(got.onOrBefore(VersionUtils.allVersions().get(0))); @@ -71,8 +71,8 @@ public class VersionUtilsTests extends ESTestCase { assertEquals(got, VersionUtils.getFirstVersion()); got = VersionUtils.randomVersionBetween(random(), Version.CURRENT, Version.CURRENT); assertEquals(got, Version.CURRENT); - got = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, Version.V_5_0_0); - assertEquals(got, Version.V_5_0_0); + got = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0_alpha1, Version.V_5_0_0_alpha1); + assertEquals(got, Version.V_5_0_0_alpha1); // implicit range of one got = VersionUtils.randomVersionBetween(random(), null, VersionUtils.getFirstVersion()); From 69b71e66f363c7ca3cb81f77fc0e9c8811196dc8 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Thu, 24 Mar 2016 08:49:36 +0100 Subject: [PATCH 40/45] Fix compilation error --- .../java/org/elasticsearch/index/mapper/core/TypeParsers.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java index 7fef5354e1a..012a72743b1 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java @@ -265,7 +265,7 @@ public class TypeParsers { iterator.remove(); } else if (propName.equals("fielddata") && propNode instanceof Map - && parserContext.indexVersionCreated().before(Version.V_5_0_0)) { + && parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1)) { // ignore for bw compat iterator.remove(); } else if (propName.equals("copy_to")) { From da42f199bda6a1f940d5b64fd57939ff84de3049 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Wed, 23 Mar 2016 10:47:37 +0100 Subject: [PATCH 41/45] Enforce isolated mode for all plugins This commit removes the isolated option, each plugin have its own classloader. --- .../plugin/PluginPropertiesExtension.groovy | 3 -- .../gradle/plugin/PluginPropertiesTask.groovy | 7 ---- .../resources/plugin-descriptor.properties | 9 ----- .../plugins/DummyPluginInfo.java | 6 +-- .../plugins/InstallPluginCommand.java | 12 ++---- .../org/elasticsearch/plugins/PluginInfo.java | 21 ++-------- .../elasticsearch/plugins/PluginsService.java | 16 ++------ .../plugins/PluginInfoTests.java | 11 +++--- .../migration/migrate_5_0/plugins.asciidoc | 4 ++ modules/build.gradle | 6 --- .../plugins/InstallPluginCommandTests.java | 38 +------------------ 11 files changed, 23 insertions(+), 110 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy index 7b949b3e1da..54699a52e76 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy @@ -39,9 +39,6 @@ class PluginPropertiesExtension { @Input String classname - @Input - boolean isolated = true - PluginPropertiesExtension(Project project) { name = project.name version = project.version diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy index b5128817fb0..856c9255312 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy @@ -54,12 +54,6 @@ class PluginPropertiesTask extends Copy { if (extension.classname == null) { throw new InvalidUserDataException('classname is a required setting for esplugin') } - doFirst { - if (extension.isolated == false) { - String warning = "WARNING: Disabling plugin isolation in ${project.path} is deprecated and will be removed in the future" - logger.warn("${'=' * warning.length()}\n${warning}\n${'=' * warning.length()}") - } - } // configure property substitution from(templateFile) into(generatedResourcesDir) @@ -80,7 +74,6 @@ class PluginPropertiesTask extends Copy { 'version': stringSnap(extension.version), 'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch), 'javaVersion': project.targetCompatibility as String, - 'isolated': extension.isolated as String, 'classname': extension.classname ] } diff --git a/buildSrc/src/main/resources/plugin-descriptor.properties b/buildSrc/src/main/resources/plugin-descriptor.properties index 7659f11bd09..ebde46d326b 100644 --- a/buildSrc/src/main/resources/plugin-descriptor.properties +++ b/buildSrc/src/main/resources/plugin-descriptor.properties @@ -38,12 +38,3 @@ java.version=${javaVersion} # # 'elasticsearch.version' version of elasticsearch compiled against elasticsearch.version=${elasticsearchVersion} -# -### deprecated elements for jvm plugins : -# -# 'isolated': true if the plugin should have its own classloader. -# passing false is deprecated, and only intended to support plugins -# that have hard dependencies against each other. If this is -# not specified, then the plugin is isolated by default. -isolated=${isolated} -# \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java b/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java index a7d088ce214..5dab19581a3 100644 --- a/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java +++ b/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java @@ -20,9 +20,9 @@ package org.elasticsearch.plugins; public class DummyPluginInfo extends PluginInfo { - private DummyPluginInfo(String name, String description, String version, String classname, boolean isolated) { - super(name, description, version, classname, isolated); + private DummyPluginInfo(String name, String description, String version, String classname) { + super(name, description, version, classname); } - public static final DummyPluginInfo INSTANCE = new DummyPluginInfo("dummy_plugin_name", "dummy plugin description", "dummy_plugin_version", "DummyPluginName", true); + public static final DummyPluginInfo INSTANCE = new DummyPluginInfo("dummy_plugin_name", "dummy plugin description", "dummy_plugin_version", "DummyPluginName"); } diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index da184b81684..4096ad57885 100644 --- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -342,7 +342,7 @@ class InstallPluginCommand extends Command { } // check for jar hell before any copying - jarHellCheck(pluginRoot, env.pluginsFile(), info.isIsolated()); + jarHellCheck(pluginRoot, env.pluginsFile()); // read optional security policy (extra permissions) // if it exists, confirm or warn the user @@ -355,19 +355,13 @@ class InstallPluginCommand extends Command { } /** check a candidate plugin for jar hell before installing it */ - void jarHellCheck(Path candidate, Path pluginsDir, boolean isolated) throws Exception { + void jarHellCheck(Path candidate, Path pluginsDir) throws Exception { // create list of current jars in classpath final List jars = new ArrayList<>(); jars.addAll(Arrays.asList(JarHell.parseClassPath())); // read existing bundles. this does some checks on the installation too. - List bundles = PluginsService.getPluginBundles(pluginsDir); - - // if we aren't isolated, we need to jarhellcheck against any other non-isolated plugins - // that's always the first bundle - if (isolated == false) { - jars.addAll(bundles.get(0).urls); - } + PluginsService.getPluginBundles(pluginsDir); // add plugin jars to the list Path pluginJars[] = FileSystemUtils.files(candidate, "*.jar"); diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java b/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java index 73464d054dd..bd063b3312c 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java @@ -44,14 +44,12 @@ public class PluginInfo implements Streamable, ToXContent { static final XContentBuilderString URL = new XContentBuilderString("url"); static final XContentBuilderString VERSION = new XContentBuilderString("version"); static final XContentBuilderString CLASSNAME = new XContentBuilderString("classname"); - static final XContentBuilderString ISOLATED = new XContentBuilderString("isolated"); } private String name; private String description; private String version; private String classname; - private boolean isolated; public PluginInfo() { } @@ -63,12 +61,11 @@ public class PluginInfo implements Streamable, ToXContent { * @param description Its description * @param version Version number */ - PluginInfo(String name, String description, String version, String classname, boolean isolated) { + PluginInfo(String name, String description, String version, String classname) { this.name = name; this.description = description; this.version = version; this.classname = classname; - this.isolated = isolated; } /** reads (and validates) plugin metadata descriptor file */ @@ -106,13 +103,12 @@ public class PluginInfo implements Streamable, ToXContent { } JarHell.checkVersionFormat(javaVersionString); JarHell.checkJavaVersion(name, javaVersionString); - boolean isolated = Boolean.parseBoolean(props.getProperty("isolated", "true")); String classname = props.getProperty("classname"); if (classname == null) { throw new IllegalArgumentException("Property [classname] is missing for plugin [" + name + "]"); } - return new PluginInfo(name, description, version, classname, isolated); + return new PluginInfo(name, description, version, classname); } /** @@ -129,13 +125,6 @@ public class PluginInfo implements Streamable, ToXContent { return description; } - /** - * @return true if plugin has isolated classloader - */ - public boolean isIsolated() { - return isolated; - } - /** * @return plugin's classname */ @@ -162,7 +151,6 @@ public class PluginInfo implements Streamable, ToXContent { this.description = in.readString(); this.version = in.readString(); this.classname = in.readString(); - this.isolated = in.readBoolean(); } @Override @@ -171,7 +159,6 @@ public class PluginInfo implements Streamable, ToXContent { out.writeString(description); out.writeString(version); out.writeString(classname); - out.writeBoolean(isolated); } @Override @@ -181,7 +168,6 @@ public class PluginInfo implements Streamable, ToXContent { builder.field(Fields.VERSION, version); builder.field(Fields.DESCRIPTION, description); builder.field(Fields.CLASSNAME, classname); - builder.field(Fields.ISOLATED, isolated); builder.endObject(); return builder; @@ -212,8 +198,7 @@ public class PluginInfo implements Streamable, ToXContent { .append("Name: ").append(name).append("\n") .append("Description: ").append(description).append("\n") .append("Version: ").append(version).append("\n") - .append(" * Classname: ").append(classname).append("\n") - .append(" * Isolated: ").append(isolated); + .append(" * Classname: ").append(classname); return information.toString(); } diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java index cf953cd1529..c1f3043cd32 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -103,7 +103,7 @@ public class PluginsService extends AbstractComponent { // first we load plugins that are on the classpath. this is for tests and transport clients for (Class pluginClass : classpathPlugins) { Plugin plugin = loadPlugin(pluginClass, settings); - PluginInfo pluginInfo = new PluginInfo(plugin.name(), plugin.description(), "NA", pluginClass.getName(), false); + PluginInfo pluginInfo = new PluginInfo(plugin.name(), plugin.description(), "NA", pluginClass.getName()); if (logger.isTraceEnabled()) { logger.trace("plugin loaded from classpath [{}]", pluginInfo); } @@ -302,9 +302,6 @@ public class PluginsService extends AbstractComponent { continue; // skip over .DS_Store etc } PluginInfo info = PluginInfo.readFromProperties(module); - if (!info.isIsolated()) { - throw new IllegalStateException("modules must be isolated: " + info); - } Bundle bundle = new Bundle(); bundle.plugins.add(info); // gather urls for jar files @@ -329,8 +326,6 @@ public class PluginsService extends AbstractComponent { } List bundles = new ArrayList<>(); - // a special purgatory for plugins that directly depend on each other - bundles.add(new Bundle()); try (DirectoryStream stream = Files.newDirectoryStream(pluginsDirectory)) { for (Path plugin : stream) { @@ -354,13 +349,8 @@ public class PluginsService extends AbstractComponent { urls.add(jar.toRealPath().toUri().toURL()); } } - final Bundle bundle; - if (info.isIsolated() == false) { - bundle = bundles.get(0); // purgatory - } else { - bundle = new Bundle(); - bundles.add(bundle); - } + final Bundle bundle = new Bundle(); + bundles.add(bundle); bundle.plugins.add(info); bundle.urls.addAll(urls); } diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java index 04bff31057d..73b31b92637 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java +++ b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java @@ -46,7 +46,6 @@ public class PluginInfoTests extends ESTestCase { assertEquals("fake desc", info.getDescription()); assertEquals("1.0", info.getVersion()); assertEquals("FakePlugin", info.getClassname()); - assertTrue(info.isIsolated()); } public void testReadFromPropertiesNameMissing() throws Exception { @@ -203,11 +202,11 @@ public class PluginInfoTests extends ESTestCase { public void testPluginListSorted() { PluginsAndModules pluginsInfo = new PluginsAndModules(); - pluginsInfo.addPlugin(new PluginInfo("c", "foo", "dummy", "dummyclass", true)); - pluginsInfo.addPlugin(new PluginInfo("b", "foo", "dummy", "dummyclass", true)); - pluginsInfo.addPlugin(new PluginInfo("e", "foo", "dummy", "dummyclass", true)); - pluginsInfo.addPlugin(new PluginInfo("a", "foo", "dummy", "dummyclass", true)); - pluginsInfo.addPlugin(new PluginInfo("d", "foo", "dummy", "dummyclass", true)); + pluginsInfo.addPlugin(new PluginInfo("c", "foo", "dummy", "dummyclass")); + pluginsInfo.addPlugin(new PluginInfo("b", "foo", "dummy", "dummyclass")); + pluginsInfo.addPlugin(new PluginInfo("e", "foo", "dummy", "dummyclass")); + pluginsInfo.addPlugin(new PluginInfo("a", "foo", "dummy", "dummyclass")); + pluginsInfo.addPlugin(new PluginInfo("d", "foo", "dummy", "dummyclass")); final List infos = pluginsInfo.getPluginInfos(); List names = infos.stream().map((input) -> input.getName()).collect(Collectors.toList()); diff --git a/docs/reference/migration/migrate_5_0/plugins.asciidoc b/docs/reference/migration/migrate_5_0/plugins.asciidoc index 10268887417..46014e3bcd4 100644 --- a/docs/reference/migration/migrate_5_0/plugins.asciidoc +++ b/docs/reference/migration/migrate_5_0/plugins.asciidoc @@ -6,6 +6,10 @@ structure of the plugin ZIP archive has changed. All the plugin files must be contained in a top-level directory called `elasticsearch`. If you use the gradle build, this structure is automatically generated. +==== Plugins isolation + +`isolated` option has been removed. Each plugin will have its own classloader. + ==== Site plugins removed Site plugins have been removed. Site plugins should be reimplemented as Kibana diff --git a/modules/build.gradle b/modules/build.gradle index 3cafe7d903f..558163a420a 100644 --- a/modules/build.gradle +++ b/modules/build.gradle @@ -35,12 +35,6 @@ subprojects { throw new InvalidModelException("Modules cannot contain config files") } - project.afterEvaluate { - if (esplugin.isolated == false) { - throw new InvalidModelException("Modules cannot disable isolation") - } - } - // these are implementation details of our build, no need to publish them! install.enabled = false uploadArchives.enabled = false diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index fdcfc44e989..106385578e9 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -193,9 +193,9 @@ public class InstallPluginCommandTests extends ESTestCase { MockTerminal terminal = new MockTerminal(); new InstallPluginCommand(env) { @Override - void jarHellCheck(Path candidate, Path pluginsDir, boolean isolated) throws Exception { + void jarHellCheck(Path candidate, Path pluginsDir) throws Exception { if (jarHellCheck) { - super.jarHellCheck(candidate, pluginsDir, isolated); + super.jarHellCheck(candidate, pluginsDir); } } }.execute(terminal, pluginUrl, true); @@ -369,40 +369,6 @@ public class InstallPluginCommandTests extends ESTestCase { assertPlugin("fake2", pluginDir2, env); } - public void testPurgatoryJarHell() throws Exception { - assumeTrue("real filesystem", isReal); - Environment environment = createEnv(fs, temp); - Path pluginDir1 = createPluginDir(temp); - PluginTestUtil.writeProperties(pluginDir1, - "description", "fake desc", - "name", "fake1", - "version", "1.0", - "elasticsearch.version", Version.CURRENT.toString(), - "java.version", System.getProperty("java.specification.version"), - "classname", "FakePlugin", - "isolated", "false"); - writeJar(pluginDir1.resolve("plugin.jar"), "FakePlugin"); - String pluginZip1 = writeZip(pluginDir1, "elasticsearch"); - installPlugin(pluginZip1, environment); - - Path pluginDir2 = createPluginDir(temp); - PluginTestUtil.writeProperties(pluginDir2, - "description", "fake desc", - "name", "fake2", - "version", "1.0", - "elasticsearch.version", Version.CURRENT.toString(), - "java.version", System.getProperty("java.specification.version"), - "classname", "FakePlugin", - "isolated", "false"); - writeJar(pluginDir2.resolve("plugin.jar"), "FakePlugin"); - String pluginZip2 = writeZip(pluginDir2, "elasticsearch"); - IllegalStateException e = expectThrows(IllegalStateException.class, () -> { - installPlugin(pluginZip2, environment, true); - }); - assertTrue(e.getMessage(), e.getMessage().contains("jar hell")); - assertInstallCleaned(environment); - } - public void testExistingPlugin() throws Exception { Environment env = createEnv(fs, temp); Path pluginDir = createPluginDir(temp); From e5074e2b1b8891eed9672645a85ff6ca706cb58c Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 24 Mar 2016 08:48:46 +0100 Subject: [PATCH 42/45] Make `parseMultiField` part of `parseField`. #17313 All our fields are supposed to support multi fields, so we could put the logic in `TypeParsers.parseField` instead of duplicating the logic in every type parser. --- .../index/mapper/core/BooleanFieldMapper.java | 2 -- .../index/mapper/core/KeywordFieldMapper.java | 2 -- .../index/mapper/core/StringFieldMapper.java | 2 -- .../index/mapper/core/TextFieldMapper.java | 2 -- .../elasticsearch/index/mapper/core/TypeParsers.java | 2 ++ .../index/mapper/geo/BaseGeoPointFieldMapper.java | 2 -- .../index/mapper/externalvalues/ExternalMapper.java | 10 ---------- 7 files changed, 2 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java index 29c37eee271..02d6a536812 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java @@ -110,8 +110,6 @@ public class BooleanFieldMapper extends FieldMapper { } builder.nullValue(lenientNodeBooleanValue(propNode)); iterator.remove(); - } else if (parseMultiField(builder, name, parserContext, propName, propNode)) { - iterator.remove(); } } return builder; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java index a3da79341a0..ab36b738c83 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java @@ -134,8 +134,6 @@ public final class KeywordFieldMapper extends FieldMapper implements AllFieldMap } else if (propName.equals("eager_global_ordinals")) { builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode)); iterator.remove(); - } else if (parseMultiField(builder, name, parserContext, propName, propNode)) { - iterator.remove(); } } return builder; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java index d19c6e31b0a..c87efcd9218 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java @@ -334,8 +334,6 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc builder.fielddataFrequencyFilter(minFrequency, maxFrequency, minSegmentSize); DocumentMapperParser.checkNoRemainingFields(propName, frequencyFilter, parserContext.indexVersionCreated()); iterator.remove(); - } else if (parseMultiField(builder, fieldName, parserContext, propName, propNode)) { - iterator.remove(); } } return builder; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java index 582ae6aff38..0baa4564b69 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java @@ -163,8 +163,6 @@ public class TextFieldMapper extends FieldMapper implements AllFieldMapper.Inclu builder.fielddataFrequencyFilter(minFrequency, maxFrequency, minSegmentSize); DocumentMapperParser.checkNoRemainingFields(propName, frequencyFilter, parserContext.indexVersionCreated()); iterator.remove(); - } else if (parseMultiField(builder, fieldName, parserContext, propName, propNode)) { - iterator.remove(); } } return builder; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java index 012a72743b1..bc7c97bc4ff 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java @@ -268,6 +268,8 @@ public class TypeParsers { && parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1)) { // ignore for bw compat iterator.remove(); + } else if (parseMultiField(builder, name, parserContext, propName, propNode)) { + iterator.remove(); } else if (propName.equals("copy_to")) { if (parserContext.isWithinMultiField()) { if (indexVersionCreated.after(Version.V_2_1_0) || diff --git a/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java index ae181b3c98c..fec0720ce70 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java @@ -220,8 +220,6 @@ public abstract class BaseGeoPointFieldMapper extends FieldMapper implements Arr } else if (propName.equals(Names.IGNORE_MALFORMED)) { builder.ignoreMalformed(XContentMapValues.lenientNodeBooleanValue(propNode)); iterator.remove(); - } else if (parseMultiField(builder, name, parserContext, propName, propNode)) { - iterator.remove(); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java index 8c25713ce3d..e1b94967b43 100755 --- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java @@ -125,16 +125,6 @@ public class ExternalMapper extends FieldMapper { public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { ExternalMapper.Builder builder = new ExternalMapper.Builder(name, generatedValue, mapperName); parseField(builder, name, node, parserContext); - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - String propName = Strings.toUnderscoreCase(entry.getKey()); - Object propNode = entry.getValue(); - - if (parseMultiField(builder, name, parserContext, propName, propNode)) { - iterator.remove(); - } - } - return builder; } } From 9399f673d1ab951d77467ed56fddcff05e6be844 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 24 Mar 2016 10:18:11 +0100 Subject: [PATCH 43/45] percolator: Add support to extract terms from TermsQuery --- .../percolator/ExtractQueryTermsService.java | 10 ++++++++- .../ExtractQueryTermsServiceTests.java | 22 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java b/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java index 8ee9d55c312..a8669c98cd8 100644 --- a/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java +++ b/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java @@ -23,6 +23,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.PrefixCodedTerms; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; @@ -92,10 +93,17 @@ public final class ExtractQueryTermsService { * an UnsupportedQueryException is thrown. */ static Set extractQueryTerms(Query query) { - // TODO: add support for the TermsQuery when it has methods to access the actual terms it encapsulates // TODO: add support for span queries if (query instanceof TermQuery) { return Collections.singleton(((TermQuery) query).getTerm()); + } else if (query instanceof TermsQuery) { + Set terms = new HashSet<>(); + TermsQuery termsQuery = (TermsQuery) query; + PrefixCodedTerms.TermIterator iterator = termsQuery.getTermData().iterator(); + for (BytesRef term = iterator.next(); term != null; term = iterator.next()) { + terms.add(new Term(iterator.field(), term)); + } + return terms; } else if (query instanceof PhraseQuery) { Term[] terms = ((PhraseQuery) query).getTerms(); if (terms.length == 0) { diff --git a/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java b/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java index f23ec6d9595..f17a4fc664d 100644 --- a/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -102,6 +103,27 @@ public class ExtractQueryTermsServiceTests extends ESTestCase { assertThat(terms.get(0).bytes(), equalTo(termQuery.getTerm().bytes())); } + public void testExtractQueryMetadata_termsQuery() { + TermsQuery termsQuery = new TermsQuery("_field", new BytesRef("_term1"), new BytesRef("_term2")); + List terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(termsQuery)); + Collections.sort(terms); + assertThat(terms.size(), equalTo(2)); + assertThat(terms.get(0).field(), equalTo("_field")); + assertThat(terms.get(0).text(), equalTo("_term1")); + assertThat(terms.get(1).field(), equalTo("_field")); + assertThat(terms.get(1).text(), equalTo("_term2")); + + // test with different fields + termsQuery = new TermsQuery(new Term("_field1", "_term1"), new Term("_field2", "_term2")); + terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(termsQuery)); + Collections.sort(terms); + assertThat(terms.size(), equalTo(2)); + assertThat(terms.get(0).field(), equalTo("_field1")); + assertThat(terms.get(0).text(), equalTo("_term1")); + assertThat(terms.get(1).field(), equalTo("_field2")); + assertThat(terms.get(1).text(), equalTo("_term2")); + } + public void testExtractQueryMetadata_phraseQuery() { PhraseQuery phraseQuery = new PhraseQuery("_field", "_term1", "term2"); List terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(phraseQuery)); From e4e08b28ff3ee1cb5525c20e5eb7ba5bd671a597 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 24 Mar 2016 11:06:20 +0100 Subject: [PATCH 44/45] [TEST] Use to wait for the node to start - it will wait for state recovery and not return 503 --- .../test/resources/packaging/scripts/packaging_test_utils.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash index fbb86b41c87..c8c1d6870a5 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash +++ b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash @@ -390,7 +390,7 @@ wait_for_elasticsearch_status() { fi echo "Making sure elasticsearch is up..." - wget -O - --retry-connrefused --waitretry=1 --timeout=60 --tries 60 http://localhost:9200 || { + wget -O - --retry-connrefused --waitretry=1 --timeout=60 --tries 60 http://localhost:9200/_cluster/health || { echo "Looks like elasticsearch never started. Here is its log:" if [ -e "$ESLOG/elasticsearch.log" ]; then cat "$ESLOG/elasticsearch.log" From 63b754aa22c7a712fe40bd7f8e232b0775a4f203 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 24 Mar 2016 11:19:25 +0100 Subject: [PATCH 45/45] Remove positional parameter from bat file --- distribution/src/main/resources/bin/elasticsearch.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/src/main/resources/bin/elasticsearch.bat b/distribution/src/main/resources/bin/elasticsearch.bat index 4da9a5d032a..a2e5d45e1c4 100644 --- a/distribution/src/main/resources/bin/elasticsearch.bat +++ b/distribution/src/main/resources/bin/elasticsearch.bat @@ -43,6 +43,6 @@ IF ERRORLEVEL 1 ( EXIT /B %ERRORLEVEL% ) -"%JAVA_HOME%\bin\java" %JAVA_OPTS% %ES_JAVA_OPTS% %ES_PARAMS% -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" start !newparams! +"%JAVA_HOME%\bin\java" %JAVA_OPTS% %ES_JAVA_OPTS% %ES_PARAMS% -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" !newparams! ENDLOCAL