diff --git a/TESTING.asciidoc b/TESTING.asciidoc index 68740c2b42e..c1b17e7b31a 100644 --- a/TESTING.asciidoc +++ b/TESTING.asciidoc @@ -44,7 +44,7 @@ In order to run Elasticsearch from source without building a package, you can run it using Maven: ------------------------------------- -./run.sh +gradle run ------------------------------------- === Test case filtering. @@ -455,5 +455,5 @@ mvn -Dtests.coverage verify jacoco:report == Debugging from an IDE -If you want to run elasticsearch from your IDE, you should execute ./run.sh +If you want to run elasticsearch from your IDE, you should execute gradle run It opens a remote debugging port that you can connect with your IDE. diff --git a/build.gradle b/build.gradle index 49189482e6b..082c9d4d553 100644 --- a/build.gradle +++ b/build.gradle @@ -31,6 +31,9 @@ buildscript { // common maven publishing configuration subprojects { + group = 'org.elasticsearch' + version = org.elasticsearch.gradle.VersionProperties.elasticsearch + plugins.withType(NexusPlugin).whenPluginAdded { modifyPom { project { @@ -91,34 +94,12 @@ allprojects { sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = sourceCompatibility - // dependency versions that are used in more than one place - versions = [ - lucene: "${luceneVersion}", - randomizedrunner: '2.2.0', - httpclient: '4.3.6' - ] - // for eclipse hacks... isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse') } } subprojects { - repositories { - mavenCentral() - maven { - name 'sonatype-snapshots' - url 'http://oss.sonatype.org/content/repositories/snapshots/' - } - if (versions.lucene.contains('-snapshot')) { - String revision = (luceneVersion =~ /\d\.\d\.\d-snapshot-(\d+)/)[0][1] - maven { - name 'lucene-snapshots' - url "http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}" - } - } - } - // include license and notice in jars gradle.projectsEvaluated { tasks.withType(Jar) { @@ -133,7 +114,7 @@ subprojects { configurations { all { resolutionStrategy { - //failOnVersionConflict() + failOnVersionConflict() dependencySubstitution { substitute module("org.elasticsearch:rest-api-spec:${version}") with project("${projectsPrefix}:rest-api-spec") @@ -199,3 +180,5 @@ task buildSrcEclipse(type: GradleBuild) { } tasks.eclipse.dependsOn(buildSrcEclipse) +task run(dependsOn: ':distribution:run') + diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 137b28964dd..f34b4e80725 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -1,4 +1,21 @@ -import org.apache.tools.ant.filters.ReplaceTokens +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ plugins { id 'groovy' @@ -8,16 +25,13 @@ plugins { apply plugin: 'idea' apply plugin: 'eclipse' -/*idea { - project { - languageLevel = '1.8' - vcs = 'Git' - } -}*/ - group = 'org.elasticsearch.gradle' archivesBaseName = 'build-tools' +Properties props = new Properties() +props.load(project.file('version.properties').newDataInputStream()) +version = props.getProperty('elasticsearch') + repositories { mavenCentral() maven { @@ -30,8 +44,8 @@ repositories { dependencies { compile gradleApi() compile localGroovy() - compile 'com.carrotsearch.randomizedtesting:junit4-ant:2.2.0' - compile('junit:junit:4.11') { + compile "com.carrotsearch.randomizedtesting:junit4-ant:${props.getProperty('randomizedrunner')}" + compile("junit:junit:${props.getProperty('junit')}") { transitive = false } compile 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3' @@ -41,16 +55,9 @@ dependencies { compile 'de.thetaphi:forbiddenapis:2.0' } -Properties props = new Properties() -props.load(project.file('../gradle.properties').newDataInputStream()) -version = props.getProperty('version') - processResources { - inputs.file('../gradle.properties') - filter ReplaceTokens, tokens: [ - 'version': props.getProperty('version'), - 'luceneVersion': props.getProperty('luceneVersion') - ] + inputs.file('version.properties') + from 'version.properties' } extraArchive { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 65b6fcc4974..95871b63ca9 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -19,14 +19,16 @@ package org.elasticsearch.gradle import org.elasticsearch.gradle.precommit.PrecommitTasks +import org.gradle.api.artifacts.ProjectDependency import org.gradle.api.GradleException import org.gradle.api.JavaVersion import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.Task +import org.gradle.api.artifacts.dsl.RepositoryHandler import org.gradle.api.tasks.bundling.Jar import org.gradle.api.tasks.compile.JavaCompile -import org.gradle.util.VersionNumber +import org.gradle.util.GradleVersion /** * Encapsulates build configuration for elasticsearch projects. @@ -36,6 +38,7 @@ class BuildPlugin implements Plugin { @Override void apply(Project project) { globalBuildInfo(project) + configureRepositories(project) project.pluginManager.apply('java') project.pluginManager.apply('carrotsearch.randomized-testing') // these plugins add lots of info to our jars @@ -45,6 +48,8 @@ class BuildPlugin implements Plugin { project.pluginManager.apply('nebula.info-scm') project.pluginManager.apply('nebula.info-jar') + configureConfigurations(project) + project.ext.versions = VersionProperties.versions configureCompile(project) configureJarManifest(project) configureTest(project) @@ -53,17 +58,6 @@ class BuildPlugin implements Plugin { static void globalBuildInfo(Project project) { if (project.rootProject.ext.has('buildChecksDone') == false) { - // enforce gradle version - VersionNumber gradleVersion = VersionNumber.parse(project.gradle.gradleVersion) - if (gradleVersion.major < 2 || gradleVersion.major == 2 && gradleVersion.minor < 6) { - throw new GradleException('Gradle 2.6 or above is required to build elasticsearch') - } - - // enforce Java version - if (!JavaVersion.current().isJava8Compatible()) { - throw new GradleException('Java 8 or above is required to build Elasticsearch') - } - // Build debugging info println '=======================================' println 'Elasticsearch Build Hamster says Hello!' @@ -71,10 +65,57 @@ class BuildPlugin implements Plugin { println " Gradle Version : ${project.gradle.gradleVersion}" println " JDK Version : ${System.getProperty('java.runtime.version')} (${System.getProperty('java.vendor')})" println " OS Info : ${System.getProperty('os.name')} ${System.getProperty('os.version')} (${System.getProperty('os.arch')})" + + // enforce gradle version + GradleVersion minGradle = GradleVersion.version('2.8') + if (GradleVersion.current() < minGradle) { + throw new GradleException("${minGradle} or above is required to build elasticsearch") + } + + // enforce Java version + if (!JavaVersion.current().isJava8Compatible()) { + throw new GradleException('Java 8 or above is required to build Elasticsearch') + } + project.rootProject.ext.buildChecksDone = true } } + /** Makes dependencies non-transitive by default */ + static void configureConfigurations(Project project) { + + // force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself + project.configurations.compile.dependencies.all { dep -> + if (!(dep instanceof ProjectDependency) && dep.getGroup() != 'org.elasticsearch') { + dep.transitive = false + } + } + project.configurations.testCompile.dependencies.all { dep -> + if (!(dep instanceof ProjectDependency) && dep.getGroup() != 'org.elasticsearch') { + dep.transitive = false + } + } + } + + /** Adds repositores used by ES dependencies */ + static void configureRepositories(Project project) { + RepositoryHandler repos = project.repositories + repos.mavenCentral() + repos.maven { + name 'sonatype-snapshots' + url 'http://oss.sonatype.org/content/repositories/snapshots/' + } + String luceneVersion = VersionProperties.lucene + if (luceneVersion.contains('-snapshot')) { + // extract the revision number from the version with a regex matcher + String revision = (luceneVersion =~ /\w+-snapshot-(\d+)/)[0][1] + repos.maven { + name 'lucene-snapshots' + url "http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}" + } + } + } + /** Adds compiler settings to the project */ static void configureCompile(Project project) { project.afterEvaluate { @@ -91,8 +132,8 @@ class BuildPlugin implements Plugin { project.afterEvaluate { project.tasks.withType(Jar) { Jar jarTask -> manifest { - attributes('X-Compile-Elasticsearch-Version': ElasticsearchProperties.version, - 'X-Compile-Lucene-Version': ElasticsearchProperties.luceneVersion) + attributes('X-Compile-Elasticsearch-Version': VersionProperties.elasticsearch, + 'X-Compile-Lucene-Version': VersionProperties.lucene) } } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/ElasticsearchProperties.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionProperties.groovy similarity index 58% rename from buildSrc/src/main/groovy/org/elasticsearch/gradle/ElasticsearchProperties.groovy rename to buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionProperties.groovy index 8628387e3dc..c24431b4cbc 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/ElasticsearchProperties.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionProperties.groovy @@ -19,19 +19,23 @@ package org.elasticsearch.gradle /** - * Accessor for properties about the version of elasticsearch this was built with. + * Accessor for shared dependency versions used by elasticsearch, namely the elasticsearch and lucene versions. */ -class ElasticsearchProperties { - static final String version - static final String luceneVersion +class VersionProperties { + static final String elasticsearch + static final String lucene + static final Map versions = new HashMap<>() static { Properties props = new Properties() - InputStream propsStream = ElasticsearchProperties.class.getResourceAsStream('/elasticsearch.properties') + InputStream propsStream = VersionProperties.class.getResourceAsStream('/version.properties') if (propsStream == null) { - throw new RuntimeException('/elasticsearch.properties resource missing') + throw new RuntimeException('/version.properties resource missing') } props.load(propsStream) - version = props.getProperty('version') - luceneVersion = props.getProperty('luceneVersion') + elasticsearch = props.getProperty('elasticsearch') + lucene = props.getProperty('lucene') + for (String property : props.stringPropertyNames()) { + versions.put(property, props.getProperty(property)) + } } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index efc2f0a4937..58d2c2e50c6 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -20,7 +20,6 @@ package org.elasticsearch.gradle.plugin import nebula.plugin.extraconfigurations.ProvidedBasePlugin import org.elasticsearch.gradle.BuildPlugin -import org.elasticsearch.gradle.ElasticsearchProperties import org.elasticsearch.gradle.test.RestIntegTestTask import org.gradle.api.Project import org.gradle.api.Task @@ -49,7 +48,7 @@ class PluginBuildPlugin extends BuildPlugin { project.integTest.configure { dependsOn project.bundlePlugin cluster { - plugin project.name, project.bundlePlugin.outputs.files + plugin project.pluginProperties.extension.name, project.bundlePlugin.outputs.files } } } @@ -64,20 +63,18 @@ class PluginBuildPlugin extends BuildPlugin { } static void configureDependencies(Project project) { - String elasticsearchVersion = ElasticsearchProperties.version project.dependencies { - provided "org.elasticsearch:elasticsearch:${elasticsearchVersion}" - testCompile "org.elasticsearch:test-framework:${elasticsearchVersion}" + provided "org.elasticsearch:elasticsearch:${project.versions.elasticsearch}" + testCompile "org.elasticsearch:test-framework:${project.versions.elasticsearch}" // we "upgrade" these optional deps to provided for plugins, since they will run // with a full elasticsearch server that includes optional deps - // TODO: remove duplication of version here with core... - provided 'com.spatial4j:spatial4j:0.4.1' - provided 'com.vividsolutions:jts:1.13' - provided 'com.github.spullara.mustache.java:compiler:0.9.1' - provided "log4j:log4j:1.2.17" - provided "log4j:apache-log4j-extras:1.2.17" - provided "org.slf4j:slf4j-api:1.6.2" - provided 'net.java.dev.jna:jna:4.1.0' + provided "com.spatial4j:spatial4j:${project.versions.spatial4j}" + provided "com.vividsolutions:jts:${project.versions.jts}" + provided "com.github.spullara.mustache.java:compiler:${project.versions.mustache}" + provided "log4j:log4j:${project.versions.log4j}" + provided "log4j:apache-log4j-extras:${project.versions.log4j}" + provided "org.slf4j:slf4j-api:${project.versions.slf4j}" + provided "net.java.dev.jna:jna:${project.versions.jna}" } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy index 3355f653b75..51853f85e00 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy @@ -18,7 +18,7 @@ */ package org.elasticsearch.gradle.plugin -import org.elasticsearch.gradle.ElasticsearchProperties +import org.elasticsearch.gradle.VersionProperties import org.gradle.api.InvalidUserDataException import org.gradle.api.Task import org.gradle.api.tasks.Copy @@ -72,7 +72,7 @@ class PluginPropertiesTask extends Copy { 'name': extension.name, 'description': extension.description, 'version': extension.version, - 'elasticsearchVersion': ElasticsearchProperties.version, + 'elasticsearchVersion': VersionProperties.elasticsearch, 'javaVersion': project.targetCompatibility as String, 'jvm': extension.jvm as String, 'site': extension.site as String, diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index 088d6c854ba..b4b08bf4160 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -36,6 +36,9 @@ class ClusterConfiguration { @Input int transportPort = 9500 + @Input + boolean daemonize = true + @Input String jvmArgs = System.getProperty('tests.jvm.argline', '') diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 0b835872ec1..31f4cb8c257 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -18,8 +18,12 @@ */ package org.elasticsearch.gradle.test +import org.gradle.internal.jvm.Jvm + +import java.nio.file.Paths + import org.apache.tools.ant.taskdefs.condition.Os -import org.elasticsearch.gradle.ElasticsearchProperties +import org.elasticsearch.gradle.VersionProperties import org.gradle.api.DefaultTask import org.gradle.api.GradleException import org.gradle.api.InvalidUserDataException @@ -52,7 +56,7 @@ class ClusterFormationTasks { /** Adds a dependency on the given distribution */ static void configureDistributionDependency(Project project, String distro) { - String elasticsearchVersion = ElasticsearchProperties.version + String elasticsearchVersion = VersionProperties.elasticsearch String packaging = distro == 'tar' ? 'tar.gz' : distro project.configurations { elasticsearchDistro @@ -101,7 +105,7 @@ class ClusterFormationTasks { for (Map.Entry plugin : config.plugins.entrySet()) { // replace every dash followed by a character with just the uppercase character String camelName = plugin.getKey().replaceAll(/-(\w)/) { _, c -> c.toUpperCase(Locale.ROOT) } - String taskName = "${task.name}#install${camelName[0].toUpperCase(Locale.ROOT) + camelName.substring(1)}" + String taskName = "${task.name}#install${camelName[0].toUpperCase(Locale.ROOT) + camelName.substring(1)}Plugin" // delay reading the file location until execution time by wrapping in a closure within a GString String file = "${ -> new File(pluginsTmpDir, plugin.getValue().singleFile.getName()).toURI().toURL().toString() }" Object[] args = [new File(home, 'bin/plugin'), 'install', file] @@ -173,12 +177,10 @@ class ClusterFormationTasks { if (config.plugins.isEmpty()) { return setup } - // collect the files for plugins into a list, but wrap each in a closure to delay - // looking for the filename until execution time - List files = config.plugins.values().collect { plugin -> return { plugin.singleFile } } + return project.tasks.create(name: name, type: Copy, dependsOn: setup) { into pluginsTmpDir - from(*files) // spread the list into varargs + from(config.plugins.values()) } } @@ -244,17 +246,22 @@ class ClusterFormationTasks { // elasticsearch.bat is spawned as it has no daemon mode return project.tasks.create(name: name, type: DefaultTask, dependsOn: setup) << { // Fall back to Ant exec task as Gradle Exec task does not support spawning yet - ant.exec(executable: 'cmd', spawn: true, dir: cwd) { + ant.exec(executable: 'cmd', spawn: config.daemonize, dir: cwd) { esEnv.each { key, value -> env(key: key, value: value) } (['/C', 'call', esScript] + esProps).each { arg(value: it) } } esPostStartActions(ant, logger) } } else { + List esExecutable = [esScript] + if(config.daemonize) { + esExecutable.add("-d") + } + return project.tasks.create(name: name, type: Exec, dependsOn: setup) { workingDir cwd executable 'sh' - args esScript, '-d' // daemonize! + args esExecutable args esProps environment esEnv errorOutput = new ByteArrayOutputStream() @@ -279,7 +286,16 @@ class ClusterFormationTasks { onlyIf { pidFile.exists() } // the pid file won't actually be read until execution time, since the read is wrapped within an inner closure of the GString ext.pid = "${ -> pidFile.getText('UTF-8').trim()}" - commandLine new File(System.getenv('JAVA_HOME'), 'bin/jps'), '-l' + File jps + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + jps = getJpsExecutableByName("jps.exe") + } else { + jps = getJpsExecutableByName("jps") + } + if (!jps.exists()) { + throw new GradleException("jps executable not found; ensure that you're running Gradle with the JDK rather than the JRE") + } + commandLine jps, '-l' standardOutput = new ByteArrayOutputStream() doLast { String out = standardOutput.toString() @@ -297,6 +313,10 @@ class ClusterFormationTasks { } } + private static File getJpsExecutableByName(String jpsExecutableName) { + return Paths.get(Jvm.current().javaHome.toString(), "bin/" + jpsExecutableName).toFile() + } + /** Adds a task to kill an elasticsearch node with the given pidfile */ static Task configureStopTask(String name, Project project, Object depends, File pidFile) { return project.tasks.create(name: name, type: Exec, dependsOn: depends) { @@ -325,7 +345,7 @@ class ClusterFormationTasks { switch (distro) { case 'zip': case 'tar': - path = "elasticsearch-${ElasticsearchProperties.version}" + path = "elasticsearch-${VersionProperties.elasticsearch}" break; default: throw new InvalidUserDataException("Unknown distribution: ${distro}") @@ -336,4 +356,4 @@ class ClusterFormationTasks { static File pidFile(File dir) { return new File(dir, 'es.pid') } -} \ No newline at end of file +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy index 102d78e3ead..e0af8f4cc8e 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy @@ -18,7 +18,7 @@ */ package org.elasticsearch.gradle.test -import org.elasticsearch.gradle.ElasticsearchProperties +import org.elasticsearch.gradle.VersionProperties import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.tasks.Copy @@ -38,7 +38,7 @@ class RestSpecHack { restSpec } project.dependencies { - restSpec "org.elasticsearch:rest-api-spec:${ElasticsearchProperties.version}" + restSpec "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}" } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy new file mode 100644 index 00000000000..ad14dfe8672 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy @@ -0,0 +1,12 @@ +package org.elasticsearch.gradle.test + +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.TaskAction + +class RunTask extends DefaultTask { + ClusterConfiguration clusterConfig = new ClusterConfiguration(httpPort: 9200, transportPort: 9300, daemonize: false) + + RunTask() { + ClusterFormationTasks.setup(project, this, clusterConfig) + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy index ffb51ba9fd4..4ba0bba0228 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy @@ -21,7 +21,8 @@ package org.elasticsearch.gradle.test import com.carrotsearch.gradle.junit4.RandomizedTestingPlugin -import org.elasticsearch.gradle.ElasticsearchProperties +import org.elasticsearch.gradle.BuildPlugin +import org.elasticsearch.gradle.VersionProperties import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.plugins.JavaBasePlugin @@ -31,6 +32,8 @@ class StandaloneTestBasePlugin implements Plugin { @Override void apply(Project project) { + BuildPlugin.configureRepositories(project) + project.pluginManager.apply(JavaBasePlugin) project.pluginManager.apply(RandomizedTestingPlugin) @@ -42,7 +45,7 @@ class StandaloneTestBasePlugin implements Plugin { test } project.dependencies { - testCompile "org.elasticsearch:test-framework:${ElasticsearchProperties.version}" + testCompile "org.elasticsearch:test-framework:${VersionProperties.elasticsearch}" } project.eclipse { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy index 675b2894913..21bf7e9a01a 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy @@ -21,7 +21,6 @@ package org.elasticsearch.gradle.test import com.carrotsearch.gradle.junit4.RandomizedTestingTask import org.elasticsearch.gradle.BuildPlugin -import org.elasticsearch.gradle.ElasticsearchProperties import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.plugins.JavaBasePlugin diff --git a/buildSrc/src/main/resources/forbidden/core-signatures.txt b/buildSrc/src/main/resources/forbidden/core-signatures.txt index 08c548f1dcc..0f54946ea37 100644 --- a/buildSrc/src/main/resources/forbidden/core-signatures.txt +++ b/buildSrc/src/main/resources/forbidden/core-signatures.txt @@ -87,3 +87,6 @@ java.util.concurrent.Future#cancel(boolean) @defaultMessage Don't try reading from paths that are not configured in Environment, resolve from Environment instead org.elasticsearch.common.io.PathUtils#get(java.lang.String, java.lang.String[]) org.elasticsearch.common.io.PathUtils#get(java.net.URI) + +@defaultMessage Don't use deprecated Query#setBoost, wrap the query into a BoostQuery instead +org.apache.lucene.search.Query#setBoost(float) diff --git a/buildSrc/version.properties b/buildSrc/version.properties new file mode 100644 index 00000000000..864ad05c400 --- /dev/null +++ b/buildSrc/version.properties @@ -0,0 +1,20 @@ +elasticsearch = 3.0.0-SNAPSHOT +lucene = 5.4.0-snapshot-1712973 + +# optional dependencies +spatial4j = 0.5 +jts = 1.13 +mustache = 0.9.1 +jackson = 2.6.2 +log4j = 1.2.17 +slf4j = 1.6.2 +jna = 4.1.0 + + +# test dependencies +randomizedrunner = 2.2.0 +junit = 4.11 +httpclient = 4.3.6 +httpcore = 4.3.3 +commonslogging = 1.1.3 +commonscodec = 1.10 diff --git a/core/build.gradle b/core/build.gradle index 101f555374b..b6c5c032e43 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -27,25 +27,23 @@ apply plugin: 'nebula.optional-base' archivesBaseName = 'elasticsearch' -versions << [ - jackson: '2.6.2', - log4j: '1.2.17', - slf4j: '1.6.2' -] - dependencies { // lucene compile "org.apache.lucene:lucene-core:${versions.lucene}" - compile "org.apache.lucene:lucene-backward-codecs:${versions.lucene}" compile "org.apache.lucene:lucene-analyzers-common:${versions.lucene}" - compile "org.apache.lucene:lucene-queries:${versions.lucene}" - compile "org.apache.lucene:lucene-memory:${versions.lucene}" + compile "org.apache.lucene:lucene-backward-codecs:${versions.lucene}" + compile "org.apache.lucene:lucene-grouping:${versions.lucene}" compile "org.apache.lucene:lucene-highlighter:${versions.lucene}" - compile "org.apache.lucene:lucene-queryparser:${versions.lucene}" - compile "org.apache.lucene:lucene-suggest:${versions.lucene}" compile "org.apache.lucene:lucene-join:${versions.lucene}" + compile "org.apache.lucene:lucene-memory:${versions.lucene}" + compile "org.apache.lucene:lucene-misc:${versions.lucene}" + compile "org.apache.lucene:lucene-queries:${versions.lucene}" + compile "org.apache.lucene:lucene-queryparser:${versions.lucene}" + compile "org.apache.lucene:lucene-sandbox:${versions.lucene}" compile "org.apache.lucene:lucene-spatial:${versions.lucene}" + compile "org.apache.lucene:lucene-spatial3d:${versions.lucene}" + compile "org.apache.lucene:lucene-suggest:${versions.lucene}" compile 'org.elasticsearch:securesm:1.0' @@ -62,10 +60,10 @@ dependencies { // json and yaml compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" compile "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" - compile(group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: versions.jackson) { - exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind' - } + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" + compile "org.yaml:snakeyaml:1.15" // used by jackson yaml + // network stack compile 'io.netty:netty:3.10.5.Final' // compression of transport protocol @@ -76,18 +74,18 @@ dependencies { compile 'org.hdrhistogram:HdrHistogram:2.1.6' // lucene spatial - compile 'com.spatial4j:spatial4j:0.5', optional - compile 'com.vividsolutions:jts:1.13', optional + compile "com.spatial4j:spatial4j:${versions.spatial4j}", optional + compile "com.vividsolutions:jts:${versions.jts}", optional // templating - compile 'com.github.spullara.mustache.java:compiler:0.9.1', optional + compile "com.github.spullara.mustache.java:compiler:${versions.mustache}", optional // logging compile "log4j:log4j:${versions.log4j}", optional compile "log4j:apache-log4j-extras:${versions.log4j}", optional compile "org.slf4j:slf4j-api:${versions.slf4j}", optional - compile 'net.java.dev.jna:jna:4.1.0', optional + compile "net.java.dev.jna:jna:${versions.jna}", optional if (isEclipse == false || project.path == "${projectsPrefix}:core-tests") { testCompile("org.elasticsearch:test-framework:${version}") { @@ -136,4 +134,3 @@ if (isEclipse == false || project.path == "${projectsPrefix}:core-tests") { Task copyRestSpec = RestSpecHack.configureTask(project, true) integTest.dependsOn copyRestSpec } - diff --git a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java index e411139bab0..81f49055223 100644 --- a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java @@ -18,18 +18,9 @@ */ package org.apache.lucene.queries; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexReaderContext; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermContext; -import org.apache.lucene.index.TermState; -import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.index.*; +import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.DisjunctionMaxQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.InPlaceMergeSorter; import org.apache.lucene.util.ToStringUtils; @@ -37,6 +28,7 @@ import org.apache.lucene.util.ToStringUtils; import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Objects; /** * BlendedTermQuery can be used to unify term statistics across @@ -77,6 +69,10 @@ public abstract class BlendedTermQuery extends Query { @Override public Query rewrite(IndexReader reader) throws IOException { + Query rewritten = super.rewrite(reader); + if (rewritten != this) { + return rewritten; + } IndexReaderContext context = reader.getContext(); TermContext[] ctx = new TermContext[terms.length]; int[] docFreqs = new int[ctx.length]; @@ -87,9 +83,7 @@ public abstract class BlendedTermQuery extends Query { final int maxDoc = reader.maxDoc(); blend(ctx, maxDoc, reader); - Query query = topLevelQuery(terms, ctx, docFreqs, maxDoc); - query.setBoost(getBoost()); - return query; + return topLevelQuery(terms, ctx, docFreqs, maxDoc); } protected abstract Query topLevelQuery(Term[] terms, TermContext[] ctx, int[] docFreqs, int maxDoc); @@ -274,20 +268,15 @@ public abstract class BlendedTermQuery extends Query { @Override public boolean equals(Object o) { if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; BlendedTermQuery that = (BlendedTermQuery) o; - if (!Arrays.equals(equalsTerms(), that.equalsTerms())) return false; - - return true; + return Arrays.equals(equalsTerms(), that.equalsTerms()); } @Override public int hashCode() { - int result = super.hashCode(); - result = 31 * result + Arrays.hashCode(equalsTerms()); - return result; + return Objects.hash(super.hashCode(), Arrays.hashCode(equalsTerms())); } public static BlendedTermQuery booleanBlendedQuery(Term[] terms, final boolean disableCoord) { @@ -298,16 +287,16 @@ public abstract class BlendedTermQuery extends Query { return new BlendedTermQuery(terms, boosts) { @Override protected Query topLevelQuery(Term[] terms, TermContext[] ctx, int[] docFreqs, int maxDoc) { - BooleanQuery.Builder query = new BooleanQuery.Builder(); - query.setDisableCoord(disableCoord); + BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); + booleanQueryBuilder.setDisableCoord(disableCoord); for (int i = 0; i < terms.length; i++) { - TermQuery termQuery = new TermQuery(terms[i], ctx[i]); - if (boosts != null) { - termQuery.setBoost(boosts[i]); + Query query = new TermQuery(terms[i], ctx[i]); + if (boosts != null && boosts[i] != 1f) { + query = new BoostQuery(query, boosts[i]); } - query.add(termQuery, BooleanClause.Occur.SHOULD); + booleanQueryBuilder.add(query, BooleanClause.Occur.SHOULD); } - return query.build(); + return booleanQueryBuilder.build(); } }; } @@ -321,16 +310,16 @@ public abstract class BlendedTermQuery extends Query { BooleanQuery.Builder lowBuilder = new BooleanQuery.Builder(); lowBuilder.setDisableCoord(disableCoord); for (int i = 0; i < terms.length; i++) { - TermQuery termQuery = new TermQuery(terms[i], ctx[i]); - if (boosts != null) { - termQuery.setBoost(boosts[i]); + Query query = new TermQuery(terms[i], ctx[i]); + if (boosts != null && boosts[i] != 1f) { + query = new BoostQuery(query, boosts[i]); } if ((maxTermFrequency >= 1f && docFreqs[i] > maxTermFrequency) || (docFreqs[i] > (int) Math.ceil(maxTermFrequency * (float) maxDoc))) { - highBuilder.add(termQuery, BooleanClause.Occur.SHOULD); + highBuilder.add(query, BooleanClause.Occur.SHOULD); } else { - lowBuilder.add(termQuery, BooleanClause.Occur.SHOULD); + lowBuilder.add(query, BooleanClause.Occur.SHOULD); } } BooleanQuery high = highBuilder.build(); @@ -363,15 +352,15 @@ public abstract class BlendedTermQuery extends Query { return new BlendedTermQuery(terms, boosts) { @Override protected Query topLevelQuery(Term[] terms, TermContext[] ctx, int[] docFreqs, int maxDoc) { - DisjunctionMaxQuery query = new DisjunctionMaxQuery(tieBreakerMultiplier); + DisjunctionMaxQuery disMaxQuery = new DisjunctionMaxQuery(tieBreakerMultiplier); for (int i = 0; i < terms.length; i++) { - TermQuery termQuery = new TermQuery(terms[i], ctx[i]); - if (boosts != null) { - termQuery.setBoost(boosts[i]); + Query query = new TermQuery(terms[i], ctx[i]); + if (boosts != null && boosts[i] != 1f) { + query = new BoostQuery(query, boosts[i]); } - query.add(termQuery); + disMaxQuery.add(query); } - return query; + return disMaxQuery; } }; } diff --git a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java index 1e9ecf7ae6f..86982bfc949 100644 --- a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; import java.io.IOException; +import java.util.Objects; /** A {@link Query} that only matches documents that are greater than or equal * to a configured doc ID. */ @@ -43,7 +44,7 @@ public final class MinDocQuery extends Query { @Override public int hashCode() { - return 31 * super.hashCode() + minDoc; + return Objects.hash(super.hashCode(), minDoc); } @Override diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java index 3ef6e5a2c6d..fce58d2f88f 100644 --- a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java +++ b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java @@ -23,13 +23,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.DisjunctionMaxQuery; -import org.apache.lucene.search.FuzzyQuery; -import org.apache.lucene.search.MatchNoDocsQuery; -import org.apache.lucene.search.MultiPhraseQuery; -import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.search.Query; +import org.apache.lucene.search.*; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.common.lucene.search.Queries; @@ -41,12 +35,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.support.QueryParsers; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.common.lucene.search.Queries.fixNegativeQueryIfNeeded; @@ -148,8 +137,7 @@ public class MapperQueryParser extends QueryParser { Query q = getFieldQuerySingle(mField, queryText, quoted); if (q != null) { added = true; - applyBoost(mField, q); - disMaxQuery.add(q); + disMaxQuery.add(applyBoost(mField, q)); } } if (!added) { @@ -161,8 +149,7 @@ public class MapperQueryParser extends QueryParser { for (String mField : fields) { Query q = getFieldQuerySingle(mField, queryText, quoted); if (q != null) { - applyBoost(mField, q); - clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD)); + clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); } } if (clauses.size() == 0) // happens for stopwords @@ -250,9 +237,8 @@ public class MapperQueryParser extends QueryParser { Query q = super.getFieldQuery(mField, queryText, slop); if (q != null) { added = true; - applyBoost(mField, q); q = applySlop(q, slop); - disMaxQuery.add(q); + disMaxQuery.add(applyBoost(mField, q)); } } if (!added) { @@ -264,9 +250,8 @@ public class MapperQueryParser extends QueryParser { for (String mField : fields) { Query q = super.getFieldQuery(mField, queryText, slop); if (q != null) { - applyBoost(mField, q); q = applySlop(q, slop); - clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD)); + clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); } } if (clauses.size() == 0) // happens for stopwords @@ -305,8 +290,7 @@ public class MapperQueryParser extends QueryParser { Query q = getRangeQuerySingle(mField, part1, part2, startInclusive, endInclusive); if (q != null) { added = true; - applyBoost(mField, q); - disMaxQuery.add(q); + disMaxQuery.add(applyBoost(mField, q)); } } if (!added) { @@ -318,8 +302,7 @@ public class MapperQueryParser extends QueryParser { for (String mField : fields) { Query q = getRangeQuerySingle(mField, part1, part2, startInclusive, endInclusive); if (q != null) { - applyBoost(mField, q); - clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD)); + clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); } } if (clauses.size() == 0) // happens for stopwords @@ -371,8 +354,7 @@ public class MapperQueryParser extends QueryParser { Query q = getFuzzyQuerySingle(mField, termStr, minSimilarity); if (q != null) { added = true; - applyBoost(mField, q); - disMaxQuery.add(q); + disMaxQuery.add(applyBoost(mField, q)); } } if (!added) { @@ -383,8 +365,9 @@ public class MapperQueryParser extends QueryParser { List clauses = new ArrayList<>(); for (String mField : fields) { Query q = getFuzzyQuerySingle(mField, termStr, minSimilarity); - applyBoost(mField, q); - clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD)); + if (q != null) { + clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); + } } return getBooleanQuery(clauses, true); } @@ -434,8 +417,7 @@ public class MapperQueryParser extends QueryParser { Query q = getPrefixQuerySingle(mField, termStr); if (q != null) { added = true; - applyBoost(mField, q); - disMaxQuery.add(q); + disMaxQuery.add(applyBoost(mField, q)); } } if (!added) { @@ -447,8 +429,7 @@ public class MapperQueryParser extends QueryParser { for (String mField : fields) { Query q = getPrefixQuerySingle(mField, termStr); if (q != null) { - applyBoost(mField, q); - clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD)); + clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); } } if (clauses.size() == 0) // happens for stopwords @@ -566,8 +547,7 @@ public class MapperQueryParser extends QueryParser { Query q = getWildcardQuerySingle(mField, termStr); if (q != null) { added = true; - applyBoost(mField, q); - disMaxQuery.add(q); + disMaxQuery.add(applyBoost(mField, q)); } } if (!added) { @@ -579,8 +559,7 @@ public class MapperQueryParser extends QueryParser { for (String mField : fields) { Query q = getWildcardQuerySingle(mField, termStr); if (q != null) { - applyBoost(mField, q); - clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD)); + clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); } } if (clauses.size() == 0) // happens for stopwords @@ -697,8 +676,7 @@ public class MapperQueryParser extends QueryParser { Query q = getRegexpQuerySingle(mField, termStr); if (q != null) { added = true; - applyBoost(mField, q); - disMaxQuery.add(q); + disMaxQuery.add(applyBoost(mField, q)); } } if (!added) { @@ -710,8 +688,7 @@ public class MapperQueryParser extends QueryParser { for (String mField : fields) { Query q = getRegexpQuerySingle(mField, termStr); if (q != null) { - applyBoost(mField, q); - clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD)); + clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); } } if (clauses.size() == 0) // happens for stopwords @@ -761,11 +738,12 @@ public class MapperQueryParser extends QueryParser { return fixNegativeQueryIfNeeded(q); } - private void applyBoost(String field, Query q) { + private Query applyBoost(String field, Query q) { Float fieldBoost = settings.fieldsAndWeights().get(field); - if (fieldBoost != null) { - q.setBoost(fieldBoost); + if (fieldBoost != null && fieldBoost != 1f) { + return new BoostQuery(q, fieldBoost); } + return q; } private Query applySlop(Query q, int slop) { @@ -779,7 +757,9 @@ public class MapperQueryParser extends QueryParser { builder.add(terms[i], positions[i]); } pq = builder.build(); - pq.setBoost(q.getBoost()); + //make sure that the boost hasn't been set beforehand, otherwise we'd lose it + assert q.getBoost() == 1f; + assert q instanceof BoostQuery == false; return pq; } else if (q instanceof MultiPhraseQuery) { ((MultiPhraseQuery) q).setSlop(slop); diff --git a/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java b/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java index 97b5b716179..95657de5158 100644 --- a/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java +++ b/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java @@ -22,11 +22,7 @@ package org.apache.lucene.search.vectorhighlight; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.queries.BlendedTermQuery; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.MultiPhraseQuery; -import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.*; import org.apache.lucene.search.spans.SpanTermQuery; import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery; @@ -103,8 +99,7 @@ public class CustomFieldQuery extends FieldQuery { for (int i = 0; i < termsIdx.length; i++) { queryBuilder.add(terms.get(i)[termsIdx[i]], pos[i]); } - PhraseQuery query = queryBuilder.build(); - query.setBoost(orig.getBoost()); + Query query = queryBuilder.build(); this.flatten(query, reader, flatQueries, orig.getBoost()); } else { Term[] t = terms.get(currentPos); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java index b003f063a02..9798e189f7b 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java @@ -77,7 +77,7 @@ public class TransportRecoveryAction extends TransportBroadcastByNodeAction()); + shardResponses.put(indexName, new ArrayList<>()); } if (request.activeOnly()) { if (recoveryState.getStage() != RecoveryState.Stage.DONE) { diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index fa6b643eb69..260fd5e732d 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -64,6 +64,17 @@ public class BulkRequest extends ActionRequest implements Composite private long sizeInBytes = 0; + public BulkRequest() { + } + + /** + * Creates a bulk request caused by some other request, which is provided as an + * argument so that its headers and context can be copied to the new request + */ + public BulkRequest(ActionRequest request) { + super(request); + } + /** * Adds a list of requests to be executed. Either index or delete requests. */ diff --git a/core/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java b/core/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java index 17343e86912..b390b77504a 100644 --- a/core/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java +++ b/core/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java @@ -37,6 +37,17 @@ public class ClearScrollRequest extends ActionRequest { private List scrollIds; + public ClearScrollRequest() { + } + + /** + * Creates a clear scroll request caused by some other request, which is provided as an + * argument so that its headers and context can be copied to the new request + */ + public ClearScrollRequest(ActionRequest request) { + super(request); + } + public List getScrollIds() { return scrollIds; } diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java b/core/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java index c1ff788dae5..537d61ac558 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java @@ -46,6 +46,14 @@ public class SearchScrollRequest extends ActionRequest { this.scrollId = scrollId; } + /** + * Creates a scroll request caused by some other request, which is provided as an + * argument so that its headers and context can be copied to the new request + */ + public SearchScrollRequest(ActionRequest request) { + super(request); + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; diff --git a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java index ccaef701747..eff1e7bd13c 100644 --- a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.support.master; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.ActionFilters; @@ -29,15 +30,16 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; +import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.BaseTransportResponseHandler; import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportService; @@ -48,6 +50,19 @@ import java.util.function.Supplier; * A base class for operations that needs to be performed on the master node. */ public abstract class TransportMasterNodeAction extends HandledTransportAction { + private static final ClusterStateObserver.ChangePredicate masterNodeChangedPredicate = new ClusterStateObserver.ChangePredicate() { + @Override + public boolean apply(ClusterState previousState, ClusterState.ClusterStateStatus previousStatus, + ClusterState newState, ClusterState.ClusterStateStatus newStatus) { + // The condition !newState.nodes().masterNodeId().equals(previousState.nodes().masterNodeId()) is not sufficient as the same master node might get reelected after a disruption. + return newState.nodes().masterNodeId() != null && newState != previousState; + } + + @Override + public boolean apply(ClusterChangedEvent event) { + return event.nodesDelta().masterNodeChanged(); + } + }; protected final TransportService transportService; protected final ClusterService clusterService; @@ -75,152 +90,125 @@ public abstract class TransportMasterNodeAction listener) { - // TODO do we really need to wrap it in a listener? the handlers should be cheap - if ((listener instanceof ThreadedActionListener) == false) { - listener = new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.LISTENER, listener); - } - innerExecute(request, listener, new ClusterStateObserver(clusterService, request.masterNodeTimeout(), logger), false); + new AsyncSingleAction(request, listener).start(); } - private void innerExecute(final Request request, final ActionListener listener, final ClusterStateObserver observer, final boolean retrying) { - final ClusterState clusterState = observer.observedState(); - final DiscoveryNodes nodes = clusterState.nodes(); - if (nodes.localNodeMaster() || localExecute(request)) { - // check for block, if blocked, retry, else, execute locally - final ClusterBlockException blockException = checkBlock(request, clusterState); - if (blockException != null) { - if (!blockException.retryable()) { - listener.onFailure(blockException); - return; - } - logger.trace("can't execute due to a cluster block, retrying", blockException); - observer.waitForNextChange( - new ClusterStateObserver.Listener() { - @Override - public void onNewClusterState(ClusterState state) { - innerExecute(request, listener, observer, false); - } + class AsyncSingleAction { - @Override - public void onClusterServiceClose() { - listener.onFailure(blockException); - } + private final ActionListener listener; + private final Request request; + private volatile ClusterStateObserver observer; - @Override - public void onTimeout(TimeValue timeout) { - listener.onFailure(blockException); - } - }, new ClusterStateObserver.ValidationPredicate() { - @Override - protected boolean validate(ClusterState newState) { - ClusterBlockException blockException = checkBlock(request, newState); - return (blockException == null || !blockException.retryable()); + private final ClusterStateObserver.ChangePredicate retryableOrNoBlockPredicate = new ClusterStateObserver.ValidationPredicate() { + @Override + protected boolean validate(ClusterState newState) { + ClusterBlockException blockException = checkBlock(request, newState); + return (blockException == null || !blockException.retryable()); + } + }; + + AsyncSingleAction(Request request, ActionListener listener) { + this.request = request; + // TODO do we really need to wrap it in a listener? the handlers should be cheap + if ((listener instanceof ThreadedActionListener) == false) { + listener = new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.LISTENER, listener); + } + this.listener = listener; + } + + public void start() { + this.observer = new ClusterStateObserver(clusterService, request.masterNodeTimeout(), logger); + doStart(); + } + + protected void doStart() { + final ClusterState clusterState = observer.observedState(); + final DiscoveryNodes nodes = clusterState.nodes(); + if (nodes.localNodeMaster() || localExecute(request)) { + // check for block, if blocked, retry, else, execute locally + final ClusterBlockException blockException = checkBlock(request, clusterState); + if (blockException != null) { + if (!blockException.retryable()) { + listener.onFailure(blockException); + } else { + logger.trace("can't execute due to a cluster block, retrying", blockException); + retry(blockException, retryableOrNoBlockPredicate); + } + } else { + ActionListener delegate = new ActionListener() { + @Override + public void onResponse(Response response) { + listener.onResponse(response); + } + + @Override + public void onFailure(Throwable t) { + if (t instanceof Discovery.FailedToCommitClusterStateException + || (t instanceof NotMasterException)) { + logger.debug("master could not publish cluster state or stepped down before publishing action [{}], scheduling a retry", t, actionName); + retry(t, masterNodeChangedPredicate); + } else { + listener.onFailure(t); } } - ); - + }; + threadPool.executor(executor).execute(new ActionRunnable(delegate) { + @Override + protected void doRun() throws Exception { + masterOperation(request, clusterService.state(), delegate); + } + }); + } } else { - threadPool.executor(executor).execute(new ActionRunnable(listener) { - @Override - protected void doRun() throws Exception { - masterOperation(request, clusterService.state(), listener); - } - }); - } - } else { - if (nodes.masterNode() == null) { - if (retrying) { - listener.onFailure(new MasterNotDiscoveredException()); - } else { + if (nodes.masterNode() == null) { logger.debug("no known master node, scheduling a retry"); - observer.waitForNextChange( - new ClusterStateObserver.Listener() { - @Override - public void onNewClusterState(ClusterState state) { - innerExecute(request, listener, observer, true); - } + retry(new MasterNotDiscoveredException(), masterNodeChangedPredicate); + } else { + transportService.sendRequest(nodes.masterNode(), actionName, request, new ActionListenerResponseHandler(listener) { + @Override + public Response newInstance() { + return newResponse(); + } - @Override - public void onClusterServiceClose() { - listener.onFailure(new NodeClosedException(clusterService.localNode())); - } - - @Override - public void onTimeout(TimeValue timeout) { - listener.onFailure(new MasterNotDiscoveredException("waited for [" + timeout + "]")); - } - }, new ClusterStateObserver.ChangePredicate() { - @Override - public boolean apply(ClusterState previousState, ClusterState.ClusterStateStatus previousStatus, - ClusterState newState, ClusterState.ClusterStateStatus newStatus) { - return newState.nodes().masterNodeId() != null; - } - - @Override - public boolean apply(ClusterChangedEvent event) { - return event.nodesDelta().masterNodeChanged(); - } + @Override + public void handleException(final TransportException exp) { + Throwable cause = exp.unwrapCause(); + if (cause instanceof ConnectTransportException) { + // we want to retry here a bit to see if a new master is elected + logger.debug("connection exception while trying to forward request with action name [{}] to master node [{}], scheduling a retry. Error: [{}]", + actionName, nodes.masterNode(), exp.getDetailedMessage()); + retry(cause, masterNodeChangedPredicate); + } else { + listener.onFailure(exp); } - ); + } + }); } - return; } - processBeforeDelegationToMaster(request, clusterState); - transportService.sendRequest(nodes.masterNode(), actionName, request, new BaseTransportResponseHandler() { - @Override - public Response newInstance() { - return newResponse(); - } + } - @Override - public void handleResponse(Response response) { - listener.onResponse(response); - } - - @Override - public String executor() { - return ThreadPool.Names.SAME; - } - - @Override - public void handleException(final TransportException exp) { - if (exp.unwrapCause() instanceof ConnectTransportException) { - // we want to retry here a bit to see if a new master is elected - logger.debug("connection exception while trying to forward request to master node [{}], scheduling a retry. Error: [{}]", - nodes.masterNode(), exp.getDetailedMessage()); - observer.waitForNextChange(new ClusterStateObserver.Listener() { - @Override - public void onNewClusterState(ClusterState state) { - innerExecute(request, listener, observer, false); - } - - @Override - public void onClusterServiceClose() { - listener.onFailure(new NodeClosedException(clusterService.localNode())); - } - - @Override - public void onTimeout(TimeValue timeout) { - listener.onFailure(new MasterNotDiscoveredException()); - } - }, new ClusterStateObserver.EventPredicate() { - @Override - public boolean apply(ClusterChangedEvent event) { - return event.nodesDelta().masterNodeChanged(); - } - } - ); - } else { - listener.onFailure(exp); + private void retry(final Throwable failure, final ClusterStateObserver.ChangePredicate changePredicate) { + observer.waitForNextChange( + new ClusterStateObserver.Listener() { + @Override + public void onNewClusterState(ClusterState state) { + doStart(); } - } - }); + + @Override + public void onClusterServiceClose() { + listener.onFailure(new NodeClosedException(clusterService.localNode())); + } + + @Override + public void onTimeout(TimeValue timeout) { + logger.debug("timed out while retrying [{}] after failure (timeout [{}])", failure, actionName, timeout); + listener.onFailure(failure); + } + }, changePredicate + ); } } } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java b/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java index a4e88fc5685..474ec755d46 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java @@ -74,18 +74,6 @@ final class ESPolicy extends Policy { } } - // Special handling for broken AWS code which destroys all SSL security - // REMOVE THIS when https://github.com/aws/aws-sdk-java/pull/432 is fixed - if (permission instanceof RuntimePermission && "accessClassInPackage.sun.security.ssl".equals(permission.getName())) { - for (StackTraceElement element : Thread.currentThread().getStackTrace()) { - if ("com.amazonaws.http.conn.ssl.SdkTLSSocketFactory".equals(element.getClassName()) && - "verifyMasterSecret".equals(element.getMethodName())) { - // we found the horrible method: the hack begins! - // force the aws code to back down, by throwing an exception that it catches. - rethrow(new IllegalAccessException("no amazon, you cannot do this.")); - } - } - } // otherwise defer to template + dynamic file permissions return template.implies(domain, permission) || dynamic.implies(permission); } @@ -104,20 +92,4 @@ final class ESPolicy extends Policy { // return UNSUPPORTED_EMPTY_COLLECTION since it is safe. return super.getPermissions(codesource); } - - /** - * Classy puzzler to rethrow any checked exception as an unchecked one. - */ - private static class Rethrower { - private void rethrow(Throwable t) throws T { - throw (T) t; - } - } - - /** - * Rethrows t (identical object). - */ - private void rethrow(Throwable t) { - new Rethrower().rethrow(t); - } } diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java index 31f5eb4a921..4674d143b9e 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -205,6 +205,7 @@ public class ClusterModule extends AbstractModule { registerClusterDynamicSetting(TransportService.SETTING_TRACE_LOG_EXCLUDE, Validator.EMPTY); registerClusterDynamicSetting(TransportService.SETTING_TRACE_LOG_EXCLUDE + ".*", Validator.EMPTY); registerClusterDynamicSetting(TransportCloseIndexAction.SETTING_CLUSTER_INDICES_CLOSE_ENABLE, Validator.BOOLEAN); + registerClusterDynamicSetting(ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE, Validator.INTEGER); } private void registerBuiltinIndexSettings() { @@ -325,4 +326,4 @@ public class ClusterModule extends AbstractModule { bind(NodeMappingRefreshAction.class).asEagerSingleton(); bind(MappingUpdatedAction.class).asEagerSingleton(); } -} \ No newline at end of file +} diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java index c0f1438d432..7fef94d5c17 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java @@ -51,7 +51,7 @@ abstract public class ClusterStateUpdateTask { * called when the task was rejected because the local node is no longer master */ public void onNoLongerMaster(String source) { - onFailure(source, new EsRejectedExecutionException("no longer master. source: [" + source + "]")); + onFailure(source, new NotMasterException("no longer master. source: [" + source + "]")); } /** diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NotMasterException.java b/core/src/main/java/org/elasticsearch/cluster/NotMasterException.java similarity index 87% rename from core/src/main/java/org/elasticsearch/discovery/zen/NotMasterException.java rename to core/src/main/java/org/elasticsearch/cluster/NotMasterException.java index d78d22aa983..892510418e4 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/NotMasterException.java +++ b/core/src/main/java/org/elasticsearch/cluster/NotMasterException.java @@ -16,13 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.discovery.zen; +package org.elasticsearch.cluster; /** * Thrown when a node join request or a master ping reaches a node which is not - * currently acting as a master. + * currently acting as a master or when a cluster state update task is to be executed + * on a node that is no longer master. */ public class NotMasterException extends IllegalStateException { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java index 1874a7b020b..140c4ad6692 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java @@ -118,6 +118,9 @@ public class RoutingAllocation { private boolean debugDecision = false; + private boolean hasPendingAsyncFetch = false; + + /** * Creates a new {@link RoutingAllocation} * @@ -246,4 +249,20 @@ public class RoutingAllocation { return decision; } } + + /** + * Returns true iff the current allocation run has not processed all of the in-flight or available + * shard or store fetches. Otherwise true + */ + public boolean hasPendingAsyncFetch() { + return hasPendingAsyncFetch; + } + + /** + * Sets a flag that signals that current allocation run has not processed all of the in-flight or available shard or store fetches. + * This state is anti-viral and can be reset in on allocation run. + */ + public void setHasPendingAsyncFetch() { + this.hasPendingAsyncFetch = true; + } } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java index 96f62b44229..83d05c8d249 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java @@ -118,7 +118,8 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards @Override public boolean allocateUnassigned(RoutingAllocation allocation) { - return rebalance(allocation); + final Balancer balancer = new Balancer(logger, allocation, weightFunction, threshold); + return balancer.allocateUnassigned(); } @Override @@ -313,6 +314,15 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards return delta <= (threshold + 0.001f); } + /** + * Allocates all possible unassigned shards + * @return true if the current configuration has been + * changed, otherwise false + */ + final boolean allocateUnassigned() { + return balance(true); + } + /** * Balances the nodes on the cluster model according to the weight * function. The configured threshold is the minimum delta between the @@ -328,16 +338,24 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards * changed, otherwise false */ public boolean balance() { + return balance(false); + } + + private boolean balance(boolean onlyAssign) { if (this.nodes.isEmpty()) { /* with no nodes this is pointless */ return false; } if (logger.isTraceEnabled()) { - logger.trace("Start balancing cluster"); + if (onlyAssign) { + logger.trace("Start balancing cluster"); + } else { + logger.trace("Start assigning unassigned shards"); + } } final RoutingNodes.UnassignedShards unassigned = routingNodes.unassigned().transactionBegin(); boolean changed = initialize(routingNodes, unassigned); - if (!changed && allocation.deciders().canRebalance(allocation).type() == Type.YES) { + if (onlyAssign == false && changed == false && allocation.deciders().canRebalance(allocation).type() == Type.YES) { NodeSorter sorter = newNodeSorter(); if (nodes.size() > 1) { /* skip if we only have one node */ for (String index : buildWeightOrderedIndidces(Operation.BALANCE, sorter)) { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java index 8fb65bbfe9b..003988f7bd5 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java @@ -76,7 +76,19 @@ public class ShardsAllocators extends AbstractComponent implements ShardsAllocat @Override public boolean rebalance(RoutingAllocation allocation) { - return allocator.rebalance(allocation); + if (allocation.hasPendingAsyncFetch() == false) { + /* + * see https://github.com/elastic/elasticsearch/issues/14387 + * if we allow rebalance operations while we are still fetching shard store data + * we might end up with unnecessary rebalance operations which can be super confusion/frustrating + * since once the fetches come back we might just move all the shards back again. + * Therefore we only do a rebalance if we have fetched all information. + */ + return allocator.rebalance(allocation); + } else { + logger.debug("skipping rebalance due to in-flight shard/store fetches"); + return false; + } } @Override diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java index 829a86666a7..3d68ed50d27 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java @@ -24,13 +24,16 @@ import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.settings.NodeSettingsService; /** * This {@link AllocationDecider} limits the number of shards per node on a per - * index basis. The allocator prevents a single node to hold more than - * {@value #INDEX_TOTAL_SHARDS_PER_NODE} per index during the allocation + * index or node-wide basis. The allocator prevents a single node to hold more + * than {@value #INDEX_TOTAL_SHARDS_PER_NODE} per index and + * {@value #CLUSTER_TOTAL_SHARDS_PER_NODE} globally during the allocation * process. The limits of this decider can be changed in real-time via a the * index settings API. *

@@ -50,66 +53,140 @@ public class ShardsLimitAllocationDecider extends AllocationDecider { public static final String NAME = "shards_limit"; + private volatile int clusterShardLimit; + /** * Controls the maximum number of shards per index on a single Elasticsearch * node. Negative values are interpreted as unlimited. */ public static final String INDEX_TOTAL_SHARDS_PER_NODE = "index.routing.allocation.total_shards_per_node"; + /** + * Controls the maximum number of shards per node on a global level. + * Negative values are interpreted as unlimited. + */ + public static final String CLUSTER_TOTAL_SHARDS_PER_NODE = "cluster.routing.allocation.total_shards_per_node"; + + class ApplySettings implements NodeSettingsService.Listener { + @Override + public void onRefreshSettings(Settings settings) { + Integer newClusterLimit = settings.getAsInt(CLUSTER_TOTAL_SHARDS_PER_NODE, null); + + if (newClusterLimit != null) { + logger.info("updating [{}] from [{}] to [{}]", CLUSTER_TOTAL_SHARDS_PER_NODE, + ShardsLimitAllocationDecider.this.clusterShardLimit, newClusterLimit); + ShardsLimitAllocationDecider.this.clusterShardLimit = newClusterLimit; + } + } + } @Inject - public ShardsLimitAllocationDecider(Settings settings) { + public ShardsLimitAllocationDecider(Settings settings, NodeSettingsService nodeSettingsService) { super(settings); + this.clusterShardLimit = settings.getAsInt(CLUSTER_TOTAL_SHARDS_PER_NODE, -1); + nodeSettingsService.addListener(new ApplySettings()); } @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { IndexMetaData indexMd = allocation.routingNodes().metaData().index(shardRouting.index()); - int totalShardsPerNode = indexMd.getSettings().getAsInt(INDEX_TOTAL_SHARDS_PER_NODE, -1); - if (totalShardsPerNode <= 0) { - return allocation.decision(Decision.YES, NAME, "total shard limit disabled: [%d] <= 0", totalShardsPerNode); + int indexShardLimit = indexMd.getSettings().getAsInt(INDEX_TOTAL_SHARDS_PER_NODE, -1); + // Capture the limit here in case it changes during this method's + // execution + final int clusterShardLimit = this.clusterShardLimit; + + if (indexShardLimit <= 0 && clusterShardLimit <= 0) { + return allocation.decision(Decision.YES, NAME, "total shard limit disabled: [index: %d, cluster: %d] <= 0", + indexShardLimit, clusterShardLimit); } - int nodeCount = 0; + int indexShardCount = 0; + int nodeShardCount = 0; for (ShardRouting nodeShard : node) { - if (!nodeShard.index().equals(shardRouting.index())) { - continue; - } // don't count relocating shards... if (nodeShard.relocating()) { continue; } - nodeCount++; + nodeShardCount++; + if (nodeShard.index().equals(shardRouting.index())) { + indexShardCount++; + } } - if (nodeCount >= totalShardsPerNode) { - return allocation.decision(Decision.NO, NAME, "too many shards for this index on node [%d], limit: [%d]", - nodeCount, totalShardsPerNode); + if (clusterShardLimit > 0 && nodeShardCount >= clusterShardLimit) { + return allocation.decision(Decision.NO, NAME, "too many shards for this node [%d], limit: [%d]", + nodeShardCount, clusterShardLimit); } - return allocation.decision(Decision.YES, NAME, "shard count under limit [%d] of total shards per node", totalShardsPerNode); + if (indexShardLimit > 0 && indexShardCount >= indexShardLimit) { + return allocation.decision(Decision.NO, NAME, "too many shards for this index [%s] on node [%d], limit: [%d]", + shardRouting.index(), indexShardCount, indexShardLimit); + } + return allocation.decision(Decision.YES, NAME, "shard count under index limit [%d] and node limit [%d] of total shards per node", + indexShardLimit, clusterShardLimit); } @Override public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { IndexMetaData indexMd = allocation.routingNodes().metaData().index(shardRouting.index()); - int totalShardsPerNode = indexMd.getSettings().getAsInt(INDEX_TOTAL_SHARDS_PER_NODE, -1); - if (totalShardsPerNode <= 0) { - return allocation.decision(Decision.YES, NAME, "total shard limit disabled: [%d] <= 0", totalShardsPerNode); + int indexShardLimit = indexMd.getSettings().getAsInt(INDEX_TOTAL_SHARDS_PER_NODE, -1); + // Capture the limit here in case it changes during this method's + // execution + final int clusterShardLimit = this.clusterShardLimit; + + if (indexShardLimit <= 0 && clusterShardLimit <= 0) { + return allocation.decision(Decision.YES, NAME, "total shard limit disabled: [index: %d, cluster: %d] <= 0", + indexShardLimit, clusterShardLimit); } - int nodeCount = 0; + int indexShardCount = 0; + int nodeShardCount = 0; for (ShardRouting nodeShard : node) { - if (!nodeShard.index().equals(shardRouting.index())) { - continue; - } // don't count relocating shards... if (nodeShard.relocating()) { continue; } - nodeCount++; + nodeShardCount++; + if (nodeShard.index().equals(shardRouting.index())) { + indexShardCount++; + } } - if (nodeCount > totalShardsPerNode) { - return allocation.decision(Decision.NO, NAME, "too many shards for this index on node [%d], limit: [%d]", - nodeCount, totalShardsPerNode); + // Subtle difference between the `canAllocate` and `canRemain` is that + // this checks > while canAllocate checks >= + if (clusterShardLimit > 0 && nodeShardCount > clusterShardLimit) { + return allocation.decision(Decision.NO, NAME, "too many shards for this node [%d], limit: [%d]", + nodeShardCount, clusterShardLimit); } - return allocation.decision(Decision.YES, NAME, "shard count under limit [%d] of total shards per node", totalShardsPerNode); + if (indexShardLimit > 0 && indexShardCount > indexShardLimit) { + return allocation.decision(Decision.NO, NAME, "too many shards for this index [%s] on node [%d], limit: [%d]", + shardRouting.index(), indexShardCount, indexShardLimit); + } + return allocation.decision(Decision.YES, NAME, "shard count under index limit [%d] and node limit [%d] of total shards per node", + indexShardLimit, clusterShardLimit); + } + + @Override + public Decision canAllocate(RoutingNode node, RoutingAllocation allocation) { + // Only checks the node-level limit, not the index-level + // Capture the limit here in case it changes during this method's + // execution + final int clusterShardLimit = this.clusterShardLimit; + + if (clusterShardLimit <= 0) { + return allocation.decision(Decision.YES, NAME, "total shard limit disabled: [cluster: %d] <= 0", + clusterShardLimit); + } + + int nodeShardCount = 0; + for (ShardRouting nodeShard : node) { + // don't count relocating shards... + if (nodeShard.relocating()) { + continue; + } + nodeShardCount++; + } + if (clusterShardLimit >= 0 && nodeShardCount >= clusterShardLimit) { + return allocation.decision(Decision.NO, NAME, "too many shards for this node [%d], limit: [%d]", + nodeShardCount, clusterShardLimit); + } + return allocation.decision(Decision.YES, NAME, "shard count under node limit [%d] of total shards per node", + clusterShardLimit); } } diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/support/AbstractLegacyBlobContainer.java b/core/src/main/java/org/elasticsearch/common/blobstore/support/AbstractLegacyBlobContainer.java deleted file mode 100644 index b95a7f28c58..00000000000 --- a/core/src/main/java/org/elasticsearch/common/blobstore/support/AbstractLegacyBlobContainer.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.blobstore.support; - -import org.elasticsearch.common.blobstore.BlobPath; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.Streams; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -/** - * Temporary compatibility interface. - * - * This class should be removed after S3 and Azure containers migrate to the new model - */ -@Deprecated -public abstract class AbstractLegacyBlobContainer extends AbstractBlobContainer { - - protected AbstractLegacyBlobContainer(BlobPath path) { - super(path); - } - - /** - * Creates a new {@link InputStream} for the given blob name - *

- * This method is deprecated and is used only for compatibility with older blob containers - * The new blob containers should use readBlob/writeBlob methods instead - */ - @Deprecated - protected abstract InputStream openInput(String blobName) throws IOException; - - /** - * Creates a new OutputStream for the given blob name - *

- * This method is deprecated and is used only for compatibility with older blob containers - * The new blob containers should override readBlob/writeBlob methods instead - */ - @Deprecated - protected abstract OutputStream createOutput(String blobName) throws IOException; - - @Override - public InputStream readBlob(String blobName) throws IOException { - return openInput(blobName); - } - - @Override - public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException { - try (OutputStream stream = createOutput(blobName)) { - Streams.copy(inputStream, stream); - } - } - - @Override - public void writeBlob(String blobName, BytesReference data) throws IOException { - try (OutputStream stream = createOutput(blobName)) { - data.writeTo(stream); - } - } -} diff --git a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java index a59af2c7f51..7191c96e33e 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java @@ -64,8 +64,9 @@ public final class AllTermQuery extends Query { @Override public Query rewrite(IndexReader reader) throws IOException { - if (getBoost() != 1f) { - return super.rewrite(reader); + Query rewritten = super.rewrite(reader); + if (rewritten != this) { + return rewritten; } boolean fieldExists = false; boolean hasPayloads = false; @@ -80,14 +81,10 @@ public final class AllTermQuery extends Query { } } if (fieldExists == false) { - Query rewritten = new MatchNoDocsQuery(); - rewritten.setBoost(getBoost()); - return rewritten; + return new MatchNoDocsQuery(); } if (hasPayloads == false) { - TermQuery rewritten = new TermQuery(term); - rewritten.setBoost(getBoost()); - return rewritten; + return new TermQuery(term); } return this; } diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java index 410796497d5..8b1dcd9dfcf 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java @@ -35,10 +35,7 @@ import org.elasticsearch.common.io.FastStringReader; import java.io.IOException; import java.io.Reader; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; /** * @@ -79,29 +76,17 @@ public class MoreLikeThisQuery extends Query { @Override public int hashCode() { - int result = boostTerms ? 1 : 0; - result = 31 * result + Float.floatToIntBits(boostTermsFactor); - result = 31 * result + Arrays.hashCode(likeText); - result = 31 * result + maxDocFreq; - result = 31 * result + maxQueryTerms; - result = 31 * result + maxWordLen; - result = 31 * result + minDocFreq; - result = 31 * result + minTermFrequency; - result = 31 * result + minWordLen; - result = 31 * result + Arrays.hashCode(moreLikeFields); - result = 31 * result + minimumShouldMatch.hashCode(); - result = 31 * result + (stopWords == null ? 0 : stopWords.hashCode()); - result = 31 * result + Float.floatToIntBits(getBoost()); - return result; + return Objects.hash(super.hashCode(), boostTerms, boostTermsFactor, Arrays.hashCode(likeText), + maxDocFreq, maxQueryTerms, maxWordLen, minDocFreq, minTermFrequency, minWordLen, + Arrays.hashCode(moreLikeFields), minimumShouldMatch, stopWords); } @Override public boolean equals(Object obj) { - if (obj == null || getClass() != obj.getClass()) + if (super.equals(obj) == false) { return false; + } MoreLikeThisQuery other = (MoreLikeThisQuery) obj; - if (getBoost() != other.getBoost()) - return false; if (!analyzer.equals(other.analyzer)) return false; if (boostTerms != other.boostTerms) @@ -141,6 +126,10 @@ public class MoreLikeThisQuery extends Query { @Override public Query rewrite(IndexReader reader) throws IOException { + Query rewritten = super.rewrite(reader); + if (rewritten != this) { + return rewritten; + } XMoreLikeThis mlt = new XMoreLikeThis(reader, similarity == null ? new DefaultSimilarity() : similarity); mlt.setFieldNames(moreLikeFields); @@ -179,10 +168,7 @@ public class MoreLikeThisQuery extends Query { mltQuery = Queries.applyMinimumShouldMatch((BooleanQuery) mltQuery, minimumShouldMatch); bqBuilder.add(mltQuery, BooleanClause.Occur.SHOULD); } - - BooleanQuery bq = bqBuilder.build(); - bq.setBoost(getBoost()); - return bq; + return bqBuilder.build(); } private void handleUnlike(XMoreLikeThis mlt, String[] unlikeText, Fields[] unlikeFields) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java index 3d870bc0794..662c3294151 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java @@ -120,8 +120,9 @@ public class MultiPhrasePrefixQuery extends Query { @Override public Query rewrite(IndexReader reader) throws IOException { - if (getBoost() != 1.0F) { - return super.rewrite(reader); + Query rewritten = super.rewrite(reader); + if (rewritten != this) { + return rewritten; } if (termArrays.isEmpty()) { return new MatchNoDocsQuery(); @@ -145,7 +146,6 @@ public class MultiPhrasePrefixQuery extends Query { return Queries.newMatchNoDocsQuery(); } query.add(terms.toArray(Term.class), position); - query.setBoost(getBoost()); return query.rewrite(reader); } @@ -233,10 +233,11 @@ public class MultiPhrasePrefixQuery extends Query { */ @Override public boolean equals(Object o) { - if (!(o instanceof MultiPhrasePrefixQuery)) return false; + if (super.equals(o) == false) { + return false; + } MultiPhrasePrefixQuery other = (MultiPhrasePrefixQuery) o; - return this.getBoost() == other.getBoost() - && this.slop == other.slop + return this.slop == other.slop && termArraysEquals(this.termArrays, other.termArrays) && this.positions.equals(other.positions); } @@ -246,11 +247,10 @@ public class MultiPhrasePrefixQuery extends Query { */ @Override public int hashCode() { - return Float.floatToIntBits(getBoost()) + return super.hashCode() ^ slop ^ termArraysHashCode() - ^ positions.hashCode() - ^ 0x4AC65113; + ^ positions.hashCode(); } // Breakout calculation of the termArrays hashcode diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java index ad1f97eb9cb..b7f534d2124 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java @@ -70,7 +70,7 @@ public class Queries { .build(); } - public static boolean isNegativeQuery(Query q) { + private static boolean isNegativeQuery(Query q) { if (!(q instanceof BooleanQuery)) { return false; } @@ -107,7 +107,7 @@ public class Queries { return false; } - public static BooleanQuery applyMinimumShouldMatch(BooleanQuery query, @Nullable String minimumShouldMatch) { + public static Query applyMinimumShouldMatch(BooleanQuery query, @Nullable String minimumShouldMatch) { if (minimumShouldMatch == null) { return query; } @@ -127,10 +127,13 @@ public class Queries { } builder.setMinimumNumberShouldMatch(msm); BooleanQuery bq = builder.build(); - bq.setBoost(query.getBoost()); - query = bq; + if (query.getBoost() != 1f) { + return new BoostQuery(bq, query.getBoost()); + } + return bq; + } else { + return query; } - return query; } private static Pattern spaceAroundLessThanPattern = Pattern.compile("(\\s+<\\s*)|(\\s*<\\s+)"); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java b/core/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java index 85e1899582c..53159660089 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java @@ -670,14 +670,14 @@ public final class XMoreLikeThis { float bestScore = -1; while ((scoreTerm = q.pop()) != null) { - TermQuery tq = new TermQuery(new Term(scoreTerm.topField, scoreTerm.word)); + Query tq = new TermQuery(new Term(scoreTerm.topField, scoreTerm.word)); if (boost) { if (bestScore == -1) { bestScore = (scoreTerm.score); } float myScore = (scoreTerm.score); - tq.setBoost(boostFactor * myScore / bestScore); + tq = new BoostQuery(tq, boostFactor * myScore / bestScore); } try { diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java index 210b32d5e42..3da5ae0e4ab 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java @@ -123,8 +123,9 @@ public class FiltersFunctionScoreQuery extends Query { @Override public Query rewrite(IndexReader reader) throws IOException { - if (getBoost() != 1.0F) { - return super.rewrite(reader); + Query rewritten = super.rewrite(reader); + if (rewritten != this) { + return rewritten; } Query newQ = subQuery.rewrite(reader); if (newQ == subQuery) diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java index 907d66957ac..972fb794fb5 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java @@ -71,8 +71,9 @@ public class FunctionScoreQuery extends Query { @Override public Query rewrite(IndexReader reader) throws IOException { - if (getBoost() != 1.0F) { - return super.rewrite(reader); + Query rewritten = super.rewrite(reader); + if (rewritten != this) { + return rewritten; } Query newQ = subQuery.rewrite(reader); if (newQ == subQuery) { diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java index 5f2cbaa9c84..7eb0a0e8bac 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java @@ -22,6 +22,7 @@ import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java index 3163e061692..8e337dd90c4 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java @@ -31,7 +31,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.discovery.zen.NotMasterException; +import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.*; diff --git a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java index e94102d4623..e560b4458b7 100644 --- a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java +++ b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java @@ -65,6 +65,7 @@ public abstract class PrimaryShardAllocator extends AbstractComponent { AsyncShardFetch.FetchResult shardState = fetchData(shard, allocation); if (shardState.hasData() == false) { logger.trace("{}: ignoring allocation, still fetching shard started state", shard); + allocation.setHasPendingAsyncFetch(); unassignedIterator.removeAndIgnore(); continue; } diff --git a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java index 03772f74630..bb75c523a7e 100644 --- a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java +++ b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java @@ -139,6 +139,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent { AsyncShardFetch.FetchResult shardStores = fetchData(shard, allocation); if (shardStores.hasData() == false) { logger.trace("{}: ignoring allocation, still fetching shard stores", shard); + allocation.setHasPendingAsyncFetch(); unassignedIterator.removeAndIgnore(); continue; // still fetching } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index 0308b7e2a33..80947260442 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -28,8 +28,8 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData; import org.elasticsearch.index.fielddata.plain.DisabledIndexFieldData; import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; -import org.elasticsearch.index.fielddata.plain.GeoPointBinaryDVIndexFieldData; -import org.elasticsearch.index.fielddata.plain.GeoPointDoubleArrayIndexFieldData; +import org.elasticsearch.index.fielddata.plain.AbstractGeoPointDVIndexFieldData; +import org.elasticsearch.index.fielddata.plain.GeoPointArrayIndexFieldData; import org.elasticsearch.index.fielddata.plain.IndexIndexFieldData; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; @@ -85,7 +85,7 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Clo buildersByTypeBuilder.put("short", MISSING_DOC_VALUES_BUILDER); buildersByTypeBuilder.put("int", MISSING_DOC_VALUES_BUILDER); buildersByTypeBuilder.put("long", MISSING_DOC_VALUES_BUILDER); - buildersByTypeBuilder.put("geo_point", new GeoPointDoubleArrayIndexFieldData.Builder()); + buildersByTypeBuilder.put("geo_point", new GeoPointArrayIndexFieldData.Builder()); buildersByTypeBuilder.put(ParentFieldMapper.NAME, new ParentChildIndexFieldData.Builder()); buildersByTypeBuilder.put(IndexFieldMapper.NAME, new IndexIndexFieldData.Builder()); buildersByTypeBuilder.put("binary", new DisabledIndexFieldData.Builder()); @@ -101,7 +101,7 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Clo .put("short", new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.SHORT)) .put("int", new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.INT)) .put("long", new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.LONG)) - .put("geo_point", new GeoPointBinaryDVIndexFieldData.Builder()) + .put("geo_point", new AbstractGeoPointDVIndexFieldData.Builder()) .put("binary", new BytesBinaryDVIndexFieldData.Builder()) .put(BooleanFieldMapper.CONTENT_TYPE, new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.BOOLEAN)) .immutableMap(); @@ -129,8 +129,8 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Clo .put(Tuple.tuple("long", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.LONG)) .put(Tuple.tuple("long", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) - .put(Tuple.tuple("geo_point", ARRAY_FORMAT), new GeoPointDoubleArrayIndexFieldData.Builder()) - .put(Tuple.tuple("geo_point", DOC_VALUES_FORMAT), new GeoPointBinaryDVIndexFieldData.Builder()) + .put(Tuple.tuple("geo_point", ARRAY_FORMAT), new GeoPointArrayIndexFieldData.Builder()) + .put(Tuple.tuple("geo_point", DOC_VALUES_FORMAT), new AbstractGeoPointDVIndexFieldData.Builder()) .put(Tuple.tuple("geo_point", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) .put(Tuple.tuple("binary", DOC_VALUES_FORMAT), new BytesBinaryDVIndexFieldData.Builder()) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java index fde583e68dc..175f041bd66 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java @@ -30,7 +30,7 @@ import java.util.Collections; /** */ -abstract class AbstractAtomicGeoPointFieldData implements AtomicGeoPointFieldData { +public abstract class AbstractAtomicGeoPointFieldData implements AtomicGeoPointFieldData { @Override public final SortedBinaryDocValues getBytesValues() { diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointBinaryDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java similarity index 51% rename from core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointBinaryDVIndexFieldData.java rename to core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java index 552aa5642ef..3939cd1b0e7 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointBinaryDVIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java @@ -21,11 +21,16 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.fielddata.*; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; +import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; +import org.elasticsearch.index.fielddata.FieldDataType; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MappedFieldType.Names; import org.elasticsearch.index.mapper.MapperService; @@ -34,9 +39,9 @@ import org.elasticsearch.search.MultiValueMode; import java.io.IOException; -public class GeoPointBinaryDVIndexFieldData extends DocValuesIndexFieldData implements IndexGeoPointFieldData { +public abstract class AbstractGeoPointDVIndexFieldData extends DocValuesIndexFieldData implements IndexGeoPointFieldData { - public GeoPointBinaryDVIndexFieldData(Index index, Names fieldNames, FieldDataType fieldDataType) { + AbstractGeoPointDVIndexFieldData(Index index, Names fieldNames, FieldDataType fieldDataType) { super(index, fieldNames, fieldDataType); } @@ -45,29 +50,43 @@ public class GeoPointBinaryDVIndexFieldData extends DocValuesIndexFieldData impl throw new IllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance"); } - @Override - public AtomicGeoPointFieldData load(LeafReaderContext context) { - try { - return new GeoPointBinaryDVAtomicFieldData(DocValues.getBinary(context.reader(), fieldNames.indexName())); - } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values", e); + /** + * Lucene 5.4 GeoPointFieldType + */ + public static class GeoPointDVIndexFieldData extends AbstractGeoPointDVIndexFieldData { + final boolean indexCreatedBefore2x; + + public GeoPointDVIndexFieldData(Index index, Names fieldNames, FieldDataType fieldDataType, final boolean indexCreatedBefore2x) { + super(index, fieldNames, fieldDataType); + this.indexCreatedBefore2x = indexCreatedBefore2x; + } + + @Override + public AtomicGeoPointFieldData load(LeafReaderContext context) { + try { + if (indexCreatedBefore2x) { + return new GeoPointLegacyDVAtomicFieldData(DocValues.getBinary(context.reader(), fieldNames.indexName())); + } + return new GeoPointDVAtomicFieldData(DocValues.getSortedNumeric(context.reader(), fieldNames.indexName())); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + @Override + public AtomicGeoPointFieldData loadDirect(LeafReaderContext context) throws Exception { + return load(context); } } - @Override - public AtomicGeoPointFieldData loadDirect(LeafReaderContext context) throws Exception { - return load(context); - } - public static class Builder implements IndexFieldData.Builder { - @Override public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { // Ignore breaker - final Names fieldNames = fieldType.names(); - return new GeoPointBinaryDVIndexFieldData(indexSettings.getIndex(), fieldNames, fieldType.fieldDataType()); + return new GeoPointDVIndexFieldData(indexSettings.getIndex(), fieldType.names(), fieldType.fieldDataType(), + // norelease cut over to .before(Version.V_2_2_0) once GeoPointFieldV2 is completely merged + indexSettings.getIndexVersionCreated().onOrBefore(Version.CURRENT)); } - } -} +} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java index 8e9fbf1a8f7..3b1629f7882 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java @@ -22,6 +22,7 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.CharsRefBuilder; +import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.index.IndexSettings; @@ -33,15 +34,34 @@ import org.elasticsearch.search.MultiValueMode; import java.io.IOException; abstract class AbstractIndexGeoPointFieldData extends AbstractIndexFieldData implements IndexGeoPointFieldData { + protected abstract static class BaseGeoPointTermsEnum { + protected final BytesRefIterator termsEnum; - protected static class GeoPointEnum { + protected BaseGeoPointTermsEnum(BytesRefIterator termsEnum) { + this.termsEnum = termsEnum; + } + } - private final BytesRefIterator termsEnum; + protected static class GeoPointTermsEnum extends BaseGeoPointTermsEnum { + protected GeoPointTermsEnum(BytesRefIterator termsEnum) { + super(termsEnum); + } + + public Long next() throws IOException { + final BytesRef term = termsEnum.next(); + if (term == null) { + return null; + } + return NumericUtils.prefixCodedToLong(term); + } + } + + protected static class GeoPointTermsEnumLegacy extends BaseGeoPointTermsEnum { private final GeoPoint next; private final CharsRefBuilder spare; - protected GeoPointEnum(BytesRefIterator termsEnum) { - this.termsEnum = termsEnum; + protected GeoPointTermsEnumLegacy(BytesRefIterator termsEnum) { + super(termsEnum); next = new GeoPoint(); spare = new CharsRefBuilder(); } @@ -67,7 +87,6 @@ abstract class AbstractIndexGeoPointFieldData extends AbstractIndexFieldData getChildResources() { + List resources = new ArrayList<>(); + resources.add(Accountables.namedAccountable("indexedPoints", indexedPoints)); + return Collections.unmodifiableList(resources); + } + + @Override + public MultiGeoPointValues getGeoPointValues() { + final RandomAccessOrds ords = ordinals.ordinals(); + final SortedDocValues singleOrds = DocValues.unwrapSingleton(ords); + final GeoPoint point = new GeoPoint(); + if (singleOrds != null) { + final GeoPointValues values = new GeoPointValues() { + @Override + public GeoPoint get(int docID) { + final int ord = singleOrds.getOrd(docID); + if (ord >= 0) { + return point.resetFromIndexHash(indexedPoints.get(ord)); + } + // todo: same issue as in ParentChildIndexFieldData, handle issue upstream? + return null; + } + }; + return FieldData.singleton(values, DocValues.docsWithValue(singleOrds, maxDoc)); + } + return new MultiGeoPointValues() { + @Override + public GeoPoint valueAt(int index) { + return point.resetFromIndexHash(indexedPoints.get(ords.ordAt(index))); + } + + @Override + public void setDocument(int docId) { + ords.setDocument(docId); + } + + @Override + public int count() { + return ords.cardinality(); + } + }; + } + } + + public static class Single extends GeoPointArrayAtomicFieldData { + private final LongArray indexedPoint; + private final BitSet set; + + public Single(LongArray indexedPoint, BitSet set) { + this.indexedPoint = indexedPoint; + this.set = set; + } + + @Override + public long ramBytesUsed() { + return RamUsageEstimator.NUM_BYTES_INT + indexedPoint.ramBytesUsed() + + (set == null ? 0 : set.ramBytesUsed()); + } + + @Override + public Collection getChildResources() { + List resources = new ArrayList<>(); + resources.add(Accountables.namedAccountable("indexedPoints", indexedPoint)); + if (set != null) { + resources.add(Accountables.namedAccountable("missing bitset", set)); + } + return Collections.unmodifiableList(resources); + } + + @Override + public MultiGeoPointValues getGeoPointValues() { + final GeoPoint point = new GeoPoint(); + final GeoPointValues values = new GeoPointValues() { + @Override + public GeoPoint get(int docID) { + return point.resetFromIndexHash(indexedPoint.get(docID)); + } + }; + return FieldData.singleton(values, set); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayIndexFieldData.java similarity index 52% rename from core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java rename to core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayIndexFieldData.java index 8d1bf71332a..6af339df085 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayIndexFieldData.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.LeafReader; @@ -23,12 +24,18 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.Terms; import org.apache.lucene.util.BitSet; +import org.elasticsearch.Version; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.DoubleArray; +import org.elasticsearch.common.util.LongArray; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.fielddata.*; +import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; +import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.FieldDataType; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.ordinals.Ordinals; import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder; import org.elasticsearch.index.mapper.MappedFieldType; @@ -36,24 +43,28 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.breaker.CircuitBreakerService; /** + * Loads FieldData for an array of GeoPoints supporting both long encoded points and backward compatible double arrays */ -public class GeoPointDoubleArrayIndexFieldData extends AbstractIndexGeoPointFieldData { - +public class GeoPointArrayIndexFieldData extends AbstractIndexGeoPointFieldData { private final CircuitBreakerService breakerService; + private final boolean indexCreatedBefore22; public static class Builder implements IndexFieldData.Builder { - @Override public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { - return new GeoPointDoubleArrayIndexFieldData(indexSettings, fieldType.names(), fieldType.fieldDataType(), cache, breakerService); + return new GeoPointArrayIndexFieldData(indexSettings, fieldType.names(), fieldType.fieldDataType(), cache, + // norelease change to .before(Version.V_2_2_0) once GeoPointFieldV2 is completely merged + breakerService, indexSettings.getIndexVersionCreated().onOrBefore(Version.CURRENT)); } } - public GeoPointDoubleArrayIndexFieldData(IndexSettings indexSettings, MappedFieldType.Names fieldNames, - FieldDataType fieldDataType, IndexFieldDataCache cache, CircuitBreakerService breakerService) { + public GeoPointArrayIndexFieldData(IndexSettings indexSettings, MappedFieldType.Names fieldNames, + FieldDataType fieldDataType, IndexFieldDataCache cache, CircuitBreakerService breakerService, + final boolean indexCreatedBefore22) { super(indexSettings, fieldNames, fieldDataType, cache); this.breakerService = breakerService; + this.indexCreatedBefore22 = indexCreatedBefore22; } @Override @@ -69,12 +80,66 @@ public class GeoPointDoubleArrayIndexFieldData extends AbstractIndexGeoPointFiel estimator.afterLoad(null, data.ramBytesUsed()); return data; } + return (indexCreatedBefore22 == true) ? loadLegacyFieldData(reader, estimator, terms, data) : loadFieldData22(reader, estimator, terms, data); + } + + /** + * long encoded geopoint field data + */ + private AtomicGeoPointFieldData loadFieldData22(LeafReader reader, NonEstimatingEstimator estimator, Terms terms, + AtomicGeoPointFieldData data) throws Exception { + LongArray indexedPoints = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(128); + final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_transient_overhead_ratio", + OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); + boolean success = false; + try (OrdinalsBuilder builder = new OrdinalsBuilder(reader.maxDoc(), acceptableTransientOverheadRatio)) { + final GeoPointTermsEnum iter = new GeoPointTermsEnum(builder.buildFromTerms(OrdinalsBuilder.wrapNumeric64Bit(terms.iterator()))); + Long hashedPoint; + long numTerms = 0; + while ((hashedPoint = iter.next()) != null) { + indexedPoints = BigArrays.NON_RECYCLING_INSTANCE.resize(indexedPoints, numTerms + 1); + indexedPoints.set(numTerms++, hashedPoint); + } + indexedPoints = BigArrays.NON_RECYCLING_INSTANCE.resize(indexedPoints, numTerms); + + Ordinals build = builder.build(fieldDataType.getSettings()); + RandomAccessOrds ordinals = build.ordinals(); + if (!(FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings + .MemoryStorageFormat.ORDINALS)) { + int maxDoc = reader.maxDoc(); + LongArray sIndexedPoint = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(reader.maxDoc()); + for (int i=0; i getChildResources() { List resources = new ArrayList<>(); @@ -117,7 +117,7 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AbstractAtomicG /** * Assumes unset values are marked in bitset, and docId is used as the index to the value array. */ - public static class Single extends GeoPointDoubleArrayAtomicFieldData { + public static class Single extends GeoPointArrayLegacyAtomicFieldData { private final DoubleArray lon, lat; private final BitSet set; @@ -130,9 +130,9 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AbstractAtomicG @Override public long ramBytesUsed() { - return RamUsageEstimator.NUM_BYTES_INT/*size*/ + lon.ramBytesUsed() + lat.ramBytesUsed() + (set == null ? 0 : set.ramBytesUsed()); + return RamUsageEstimator.NUM_BYTES_INT + lon.ramBytesUsed() + lat.ramBytesUsed() + (set == null ? 0 : set.ramBytesUsed()); } - + @Override public Collection getChildResources() { List resources = new ArrayList<>(); diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java new file mode 100644 index 00000000000..de66ab09d62 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java @@ -0,0 +1,90 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata.plain; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.index.fielddata.MultiGeoPointValues; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; + +final class GeoPointDVAtomicFieldData extends AbstractAtomicGeoPointFieldData { + + private final SortedNumericDocValues values; + + GeoPointDVAtomicFieldData(SortedNumericDocValues values) { + super(); + this.values = values; + } + + @Override + public long ramBytesUsed() { + return 0; // not exposed by Lucene + } + + @Override + public Collection getChildResources() { + return Collections.emptyList(); + } + + @Override + public void close() { + // no-op + } + + @Override + public MultiGeoPointValues getGeoPointValues() { + return new MultiGeoPointValues() { + GeoPoint[] points = new GeoPoint[0]; + private int count = 0; + + @Override + public void setDocument(int docId) { + values.setDocument(docId); + count = values.count(); + if (count > points.length) { + final int previousLength = points.length; + points = Arrays.copyOf(points, ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)); + for (int i = previousLength; i < points.length; ++i) { + points[i] = new GeoPoint(); + } + } + for (int i=0; i getChildResources() { return Collections.emptyList(); @@ -97,5 +97,4 @@ final class GeoPointBinaryDVAtomicFieldData extends AbstractAtomicGeoPointFieldD }; } - } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index 686cfcfe6e2..27b96b27a44 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -219,8 +219,9 @@ public class DateFieldMapper extends NumberFieldMapper { @Override public Query rewrite(IndexReader reader) throws IOException { - if (getBoost() != 1.0F) { - return super.rewrite(reader); + Query rewritten = super.rewrite(reader); + if (rewritten != this) { + return rewritten; } return innerRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser); } @@ -229,11 +230,9 @@ public class DateFieldMapper extends NumberFieldMapper { @Override public boolean equals(Object o) { if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; LateParsingQuery that = (LateParsingQuery) o; - if (includeLower != that.includeLower) return false; if (includeUpper != that.includeUpper) return false; if (lowerTerm != null ? !lowerTerm.equals(that.lowerTerm) : that.lowerTerm != null) return false; @@ -245,13 +244,7 @@ public class DateFieldMapper extends NumberFieldMapper { @Override public int hashCode() { - int result = super.hashCode(); - result = 31 * result + (lowerTerm != null ? lowerTerm.hashCode() : 0); - result = 31 * result + (upperTerm != null ? upperTerm.hashCode() : 0); - result = 31 * result + (includeLower ? 1 : 0); - result = 31 * result + (includeUpper ? 1 : 0); - result = 31 * result + (timeZone != null ? timeZone.hashCode() : 0); - return result; + return Objects.hash(super.hashCode(), lowerTerm, upperTerm, includeLower, includeUpper, timeZone); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java index 560476a69d8..1f92f53ec07 100644 --- a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java @@ -19,7 +19,10 @@ package org.elasticsearch.index.query; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.spans.SpanBoostQuery; +import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; @@ -74,7 +77,13 @@ public abstract class AbstractQueryBuilder exte public final Query toQuery(QueryShardContext context) throws IOException { Query query = doToQuery(context); if (query != null) { - setFinalBoost(query); + if (boost != DEFAULT_BOOST) { + if (query instanceof SpanQuery) { + query = new SpanBoostQuery((SpanQuery) query, boost); + } else { + query = new BoostQuery(query, boost); + } + } if (queryName != null) { context.addNamedQuery(queryName, query); } @@ -82,20 +91,6 @@ public abstract class AbstractQueryBuilder exte return query; } - /** - * Sets the main boost to the query obtained by converting the current query into a lucene query. - * The default behaviour is to set the main boost, after verifying that we are not overriding any non default boost - * value that was previously set to the lucene query. That case would require some manual decision on how to combine - * the main boost with the boost coming from lucene by overriding this method. - * @throws IllegalStateException if the lucene query boost has already been set - */ - protected void setFinalBoost(Query query) { - if (query.getBoost() != AbstractQueryBuilder.DEFAULT_BOOST) { - throw new IllegalStateException("lucene query boost is already set, override setFinalBoost to define how to combine lucene boost with main boost"); - } - query.setBoost(boost); - } - @Override public final Query toFilter(QueryShardContext context) throws IOException { Query result = null; diff --git a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java index 43b23778d56..9ed1623f66a 100644 --- a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java @@ -279,8 +279,8 @@ public class BoolQueryBuilder extends AbstractQueryBuilder { } else { minimumShouldMatch = this.minimumShouldMatch; } - booleanQuery = Queries.applyMinimumShouldMatch(booleanQuery, minimumShouldMatch); - return adjustPureNegative ? fixNegativeQueryIfNeeded(booleanQuery) : booleanQuery; + Query query = Queries.applyMinimumShouldMatch(booleanQuery, minimumShouldMatch); + return adjustPureNegative ? fixNegativeQueryIfNeeded(query) : query; } private static void addBooleanClauses(QueryShardContext context, BooleanQuery.Builder booleanQueryBuilder, List clauses, Occur occurs) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java index dfe767b3bc6..c3bd9995a8e 100644 --- a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java @@ -115,8 +115,6 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder { currentFieldName = parser.currentName(); token = parser.nextToken(); if (parseContext.parseFieldMatcher().match(currentFieldName, SHAPE_FIELD)) { - XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType()).copyCurrentStructure(parser); + XContentBuilder builder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); shape = builder.bytes(); } else if (parseContext.parseFieldMatcher().match(currentFieldName, STRATEGY_FIELD)) { String strategyName = parser.text(); diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java index f07fba8234b..a78cb6c4fd7 100644 --- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java @@ -215,8 +215,6 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder { } /** - * Get the setting for handling zero terms queries. - * @see #zeroTermsQuery(ZeroTermsQuery) + * Returns the setting for handling zero terms queries. */ public MatchQuery.ZeroTermsQuery zeroTermsQuery() { return this.zeroTermsQuery; diff --git a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java index 57e00a2dc2b..be05333a8eb 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java @@ -548,12 +548,6 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder handleFieldsMatchPattern(MapperService mapperService, Map fieldsBoosts) { Map newFieldsBoosts = new TreeMap<>(); for (Map.Entry fieldBoost : fieldsBoosts.entrySet()) { diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java index d0aa8dff9a0..393ff1e59a7 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java @@ -22,6 +22,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.queryparser.classic.MapperQueryParser; import org.apache.lucene.queryparser.classic.QueryParserSettings; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.automaton.Operations; @@ -36,10 +37,7 @@ import org.elasticsearch.index.query.support.QueryParsers; import org.joda.time.DateTimeZone; import java.io.IOException; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.TreeMap; +import java.util.*; /** * A query that parses a query string and runs it. There are two modes that this operates. The first, @@ -722,16 +720,25 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder boosts = new ArrayList<>(); + while(query instanceof BoostQuery) { + BoostQuery boostQuery = (BoostQuery) query; + boosts.add(boostQuery.getBoost()); + query = boostQuery.getQuery(); + } + query = Queries.fixNegativeQueryIfNeeded(query); if (query instanceof BooleanQuery) { query = Queries.applyMinimumShouldMatch((BooleanQuery) query, this.minimumShouldMatch()); } + + //restore the previous BoostQuery wrapping + for (int i = boosts.size() - 1; i >= 0; i--) { + query = new BoostQuery(query, boosts.get(i)); + } + return query; } - - @Override - protected void setFinalBoost(Query query) { - //we need to preserve the boost that came out of the parsing phase - query.setBoost(query.getBoost() * boost); - } } diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java index 10407a2ba5f..f69ac8c0548 100644 --- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java @@ -104,10 +104,7 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder @Override public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + Objects.hashCode(script); - return result; + return Objects.hash(super.hashCode(), script); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java index f8b0deaf9be..7627644e750 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java @@ -63,8 +63,7 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp try { Query q = createBooleanQuery(entry.getKey(), text, super.getDefaultOperator()); if (q != null) { - q.setBoost(entry.getValue()); - bq.add(q, BooleanClause.Occur.SHOULD); + bq.add(wrapWithBoost(q, entry.getValue()), BooleanClause.Occur.SHOULD); } } catch (RuntimeException e) { rethrowUnlessLenient(e); @@ -86,9 +85,8 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp bq.setDisableCoord(true); for (Map.Entry entry : weights.entrySet()) { try { - Query q = new FuzzyQuery(new Term(entry.getKey(), text), fuzziness); - q.setBoost(entry.getValue()); - bq.add(q, BooleanClause.Occur.SHOULD); + Query query = new FuzzyQuery(new Term(entry.getKey(), text), fuzziness); + bq.add(wrapWithBoost(query, entry.getValue()), BooleanClause.Occur.SHOULD); } catch (RuntimeException e) { rethrowUnlessLenient(e); } @@ -104,8 +102,7 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp try { Query q = createPhraseQuery(entry.getKey(), text, slop); if (q != null) { - q.setBoost(entry.getValue()); - bq.add(q, BooleanClause.Occur.SHOULD); + bq.add(wrapWithBoost(q, entry.getValue()), BooleanClause.Occur.SHOULD); } } catch (RuntimeException e) { rethrowUnlessLenient(e); @@ -129,12 +126,12 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp try { if (settings.analyzeWildcard()) { Query analyzedQuery = newPossiblyAnalyzedQuery(entry.getKey(), text); - analyzedQuery.setBoost(entry.getValue()); - bq.add(analyzedQuery, BooleanClause.Occur.SHOULD); + if (analyzedQuery != null) { + bq.add(wrapWithBoost(analyzedQuery, entry.getValue()), BooleanClause.Occur.SHOULD); + } } else { - PrefixQuery prefix = new PrefixQuery(new Term(entry.getKey(), text)); - prefix.setBoost(entry.getValue()); - bq.add(prefix, BooleanClause.Occur.SHOULD); + Query query = new PrefixQuery(new Term(entry.getKey(), text)); + bq.add(wrapWithBoost(query, entry.getValue()), BooleanClause.Occur.SHOULD); } } catch (RuntimeException e) { return rethrowUnlessLenient(e); @@ -143,6 +140,13 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp return super.simplify(bq.build()); } + private static Query wrapWithBoost(Query query, float boost) { + if (boost != AbstractQueryBuilder.DEFAULT_BOOST) { + return new BoostQuery(query, boost); + } + return query; + } + /** * Analyze the given string using its analyzer, constructing either a * {@code PrefixQuery} or a {@code BooleanQuery} made up diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java index 78d3bffd228..29720195c83 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java @@ -299,11 +299,6 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder((MultiTermQuery) subQuery); + SpanQuery wrapper = new SpanMultiTermQueryWrapper<>((MultiTermQuery) subQuery); + if (boost != AbstractQueryBuilder.DEFAULT_BOOST) { + wrapper = new SpanBoostQuery(wrapper, boost); + } + return wrapper; } @Override diff --git a/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java index 2cecc4e2b0d..59ff19748af 100644 --- a/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java @@ -110,11 +110,6 @@ public class TemplateQueryBuilder extends AbstractQueryBuilder()); + groups.put(actualAnalyzer, new ArrayList<>()); } Float boost = entry.getValue(); boost = boost == null ? Float.valueOf(1.0f) : boost; groups.get(actualAnalyzer).add(new FieldAndFieldType(name, fieldType, boost)); } else { - missing.add(new Tuple(name, entry.getValue())); + missing.add(new Tuple<>(name, entry.getValue())); } } diff --git a/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java b/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java index 8d7dba292f4..a2e9e1b689d 100644 --- a/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java +++ b/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.fielddata.MultiGeoPointValues; import java.io.IOException; +import java.util.Objects; /** * @@ -94,11 +95,7 @@ public class InMemoryGeoBoundingBoxQuery extends Query { @Override public int hashCode() { - int h = super.hashCode(); - h = 31 * h + fieldName().hashCode(); - h = 31 * h + topLeft.hashCode(); - h = 31 * h + bottomRight.hashCode(); - return h; + return Objects.hash(super.hashCode(), fieldName(), topLeft, bottomRight); } private static class Meridian180GeoBoundingBoxBits implements Bits { diff --git a/core/src/main/java/org/elasticsearch/index/search/nested/IncludeNestedDocsQuery.java b/core/src/main/java/org/elasticsearch/index/search/nested/IncludeNestedDocsQuery.java deleted file mode 100644 index e3631269fbe..00000000000 --- a/core/src/main/java/org/elasticsearch/index/search/nested/IncludeNestedDocsQuery.java +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.search.nested; - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Weight; -import org.apache.lucene.search.join.BitSetProducer; -import org.apache.lucene.util.BitSet; - -import java.io.IOException; -import java.util.Collection; -import java.util.Set; - -/** - * A special query that accepts a top level parent matching query, and returns the nested docs of the matching parent - * doc as well. This is handy when deleting by query, don't use it for other purposes. - * - * @elasticsearch.internal - */ -public class IncludeNestedDocsQuery extends Query { - - private final BitSetProducer parentFilter; - private final Query parentQuery; - - // If we are rewritten, this is the original childQuery we - // were passed; we use this for .equals() and - // .hashCode(). This makes rewritten query equal the - // original, so that user does not have to .rewrite() their - // query before searching: - private final Query origParentQuery; - - - public IncludeNestedDocsQuery(Query parentQuery, BitSetProducer parentFilter) { - this.origParentQuery = parentQuery; - this.parentQuery = parentQuery; - this.parentFilter = parentFilter; - } - - // For rewriting - IncludeNestedDocsQuery(Query rewrite, Query originalQuery, IncludeNestedDocsQuery previousInstance) { - this.origParentQuery = originalQuery; - this.parentQuery = rewrite; - this.parentFilter = previousInstance.parentFilter; - setBoost(previousInstance.getBoost()); - } - - // For cloning - IncludeNestedDocsQuery(Query originalQuery, IncludeNestedDocsQuery previousInstance) { - this.origParentQuery = originalQuery; - this.parentQuery = originalQuery; - this.parentFilter = previousInstance.parentFilter; - } - - @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return new IncludeNestedDocsWeight(this, parentQuery, parentQuery.createWeight(searcher, needsScores), parentFilter); - } - - static class IncludeNestedDocsWeight extends Weight { - - private final Query parentQuery; - private final Weight parentWeight; - private final BitSetProducer parentsFilter; - - IncludeNestedDocsWeight(Query query, Query parentQuery, Weight parentWeight, BitSetProducer parentsFilter) { - super(query); - this.parentQuery = parentQuery; - this.parentWeight = parentWeight; - this.parentsFilter = parentsFilter; - } - - @Override - public void extractTerms(Set terms) { - parentWeight.extractTerms(terms); - } - - @Override - public void normalize(float norm, float topLevelBoost) { - parentWeight.normalize(norm, topLevelBoost); - } - - @Override - public float getValueForNormalization() throws IOException { - return parentWeight.getValueForNormalization(); // this query is never boosted so just delegate... - } - - @Override - public Scorer scorer(LeafReaderContext context) throws IOException { - final Scorer parentScorer = parentWeight.scorer(context); - - // no matches - if (parentScorer == null) { - return null; - } - - BitSet parents = parentsFilter.getBitSet(context); - if (parents == null) { - // No matches - return null; - } - - int firstParentDoc = parentScorer.nextDoc(); - if (firstParentDoc == DocIdSetIterator.NO_MORE_DOCS) { - // No matches - return null; - } - return new IncludeNestedDocsScorer(this, parentScorer, parents, firstParentDoc); - } - - @Override - public Explanation explain(LeafReaderContext context, int doc) throws IOException { - return null; //Query is used internally and not by users, so explain can be empty - } - } - - static class IncludeNestedDocsScorer extends Scorer { - - final Scorer parentScorer; - final BitSet parentBits; - - int currentChildPointer = -1; - int currentParentPointer = -1; - int currentDoc = -1; - - IncludeNestedDocsScorer(Weight weight, Scorer parentScorer, BitSet parentBits, int currentParentPointer) { - super(weight); - this.parentScorer = parentScorer; - this.parentBits = parentBits; - this.currentParentPointer = currentParentPointer; - if (currentParentPointer == 0) { - currentChildPointer = 0; - } else { - this.currentChildPointer = this.parentBits.prevSetBit(currentParentPointer - 1); - if (currentChildPointer == -1) { - // no previous set parent, we delete from doc 0 - currentChildPointer = 0; - } else { - currentChildPointer++; // we only care about children - } - } - - currentDoc = currentChildPointer; - } - - @Override - public Collection getChildren() { - return parentScorer.getChildren(); - } - - @Override - public int nextDoc() throws IOException { - if (currentParentPointer == NO_MORE_DOCS) { - return (currentDoc = NO_MORE_DOCS); - } - - if (currentChildPointer == currentParentPointer) { - // we need to return the current parent as well, but prepare to return - // the next set of children - currentDoc = currentParentPointer; - currentParentPointer = parentScorer.nextDoc(); - if (currentParentPointer != NO_MORE_DOCS) { - currentChildPointer = parentBits.prevSetBit(currentParentPointer - 1); - if (currentChildPointer == -1) { - // no previous set parent, just set the child to the current parent - currentChildPointer = currentParentPointer; - } else { - currentChildPointer++; // we only care about children - } - } - } else { - currentDoc = currentChildPointer++; - } - - assert currentDoc != -1; - return currentDoc; - } - - @Override - public int advance(int target) throws IOException { - if (target == NO_MORE_DOCS) { - return (currentDoc = NO_MORE_DOCS); - } - - if (target == 0) { - return nextDoc(); - } - - if (target < currentParentPointer) { - currentDoc = currentParentPointer = parentScorer.advance(target); - if (currentParentPointer == NO_MORE_DOCS) { - return (currentDoc = NO_MORE_DOCS); - } - if (currentParentPointer == 0) { - currentChildPointer = 0; - } else { - currentChildPointer = parentBits.prevSetBit(currentParentPointer - 1); - if (currentChildPointer == -1) { - // no previous set parent, just set the child to 0 to delete all up to the parent - currentChildPointer = 0; - } else { - currentChildPointer++; // we only care about children - } - } - } else { - currentDoc = currentChildPointer++; - } - - return currentDoc; - } - - @Override - public float score() throws IOException { - return parentScorer.score(); - } - - @Override - public int freq() throws IOException { - return parentScorer.freq(); - } - - @Override - public int docID() { - return currentDoc; - } - - @Override - public long cost() { - return parentScorer.cost(); - } - } - - @Override - public Query rewrite(IndexReader reader) throws IOException { - final Query parentRewrite = parentQuery.rewrite(reader); - if (parentRewrite != parentQuery) { - return new IncludeNestedDocsQuery(parentRewrite, parentQuery, this); - } else { - return this; - } - } - - @Override - public String toString(String field) { - return "IncludeNestedDocsQuery (" + parentQuery.toString() + ")"; - } - - @Override - public boolean equals(Object _other) { - if (_other instanceof IncludeNestedDocsQuery) { - final IncludeNestedDocsQuery other = (IncludeNestedDocsQuery) _other; - return origParentQuery.equals(other.origParentQuery) && parentFilter.equals(other.parentFilter); - } else { - return false; - } - } - - @Override - public int hashCode() { - final int prime = 31; - int hash = 1; - hash = prime * hash + origParentQuery.hashCode(); - hash = prime * hash + parentFilter.hashCode(); - return hash; - } - - @Override - public Query clone() { - Query clonedQuery = origParentQuery.clone(); - return new IncludeNestedDocsQuery(clonedQuery, this); - } -} diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java index dff59e9b244..5603001a293 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java @@ -27,7 +27,6 @@ import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.engine.EngineConfig; import java.io.IOException; @@ -54,13 +53,11 @@ public class IndexSearcherWrapper { } /** - * @param engineConfig The engine config which can be used to get the query cache and query cache policy from - * when creating a new index searcher * @param searcher The provided index searcher to be wrapped to add custom functionality * @return a new index searcher wrapping the provided index searcher or if no wrapping was performed * the provided index searcher */ - protected IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws IOException { + protected IndexSearcher wrap(IndexSearcher searcher) throws IOException { return searcher; } /** @@ -69,7 +66,7 @@ public class IndexSearcherWrapper { * * This is invoked each time a {@link Engine.Searcher} is requested to do an operation. (for example search) */ - public final Engine.Searcher wrap(EngineConfig engineConfig, Engine.Searcher engineSearcher) throws IOException { + public final Engine.Searcher wrap(Engine.Searcher engineSearcher) throws IOException { final ElasticsearchDirectoryReader elasticsearchDirectoryReader = ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(engineSearcher.getDirectoryReader()); if (elasticsearchDirectoryReader == null) { throw new IllegalStateException("Can't wrap non elasticsearch directory reader"); @@ -87,14 +84,15 @@ public class IndexSearcherWrapper { } } + final IndexSearcher origIndexSearcher = engineSearcher.searcher(); final IndexSearcher innerIndexSearcher = new IndexSearcher(reader); - innerIndexSearcher.setQueryCache(engineConfig.getQueryCache()); - innerIndexSearcher.setQueryCachingPolicy(engineConfig.getQueryCachingPolicy()); - innerIndexSearcher.setSimilarity(engineConfig.getSimilarity()); + innerIndexSearcher.setQueryCache(origIndexSearcher.getQueryCache()); + innerIndexSearcher.setQueryCachingPolicy(origIndexSearcher.getQueryCachingPolicy()); + innerIndexSearcher.setSimilarity(origIndexSearcher.getSimilarity(true)); // TODO: Right now IndexSearcher isn't wrapper friendly, when it becomes wrapper friendly we should revise this extension point // For example if IndexSearcher#rewrite() is overwritten than also IndexSearcher#createNormalizedWeight needs to be overwritten // This needs to be fixed before we can allow the IndexSearcher from Engine to be wrapped multiple times - final IndexSearcher indexSearcher = wrap(engineConfig, innerIndexSearcher); + final IndexSearcher indexSearcher = wrap(innerIndexSearcher); if (reader == nonClosingReaderWrapper && indexSearcher == innerIndexSearcher) { return engineSearcher; } else { diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index aad1497c9dc..9c7cf499e50 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -727,7 +727,7 @@ public class IndexShard extends AbstractIndexShardComponent { final Engine.Searcher searcher = engine.acquireSearcher(source); boolean success = false; try { - final Engine.Searcher wrappedSearcher = searcherWrapper == null ? searcher : searcherWrapper.wrap(engineConfig, searcher); + final Engine.Searcher wrappedSearcher = searcherWrapper == null ? searcher : searcherWrapper.wrap(searcher); assert wrappedSearcher != null; success = true; return wrappedSearcher; diff --git a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java index 12cf8652ccb..33f3c127d67 100644 --- a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java +++ b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.node.settings.NodeSettingsService; @@ -40,6 +41,8 @@ import java.util.concurrent.atomic.AtomicLong; */ public class HierarchyCircuitBreakerService extends CircuitBreakerService { + private static final String CHILD_LOGGER_PREFIX = "org.elasticsearch.indices.breaker."; + private final ConcurrentMap breakers = new ConcurrentHashMap(); // Old pre-1.4.0 backwards compatible settings @@ -237,7 +240,8 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService { } else { CircuitBreaker oldBreaker; CircuitBreaker breaker = new ChildMemoryCircuitBreaker(breakerSettings, - logger, this, breakerSettings.getName()); + Loggers.getLogger(CHILD_LOGGER_PREFIX + breakerSettings.getName()), + this, breakerSettings.getName()); for (;;) { oldBreaker = breakers.putIfAbsent(breakerSettings.getName(), breaker); @@ -245,7 +249,9 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService { return; } breaker = new ChildMemoryCircuitBreaker(breakerSettings, - (ChildMemoryCircuitBreaker)oldBreaker, logger, this, breakerSettings.getName()); + (ChildMemoryCircuitBreaker)oldBreaker, + Loggers.getLogger(CHILD_LOGGER_PREFIX + breakerSettings.getName()), + this, breakerSettings.getName()); if (breakers.replace(breakerSettings.getName(), oldBreaker, breaker)) { return; diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginManager.java b/core/src/main/java/org/elasticsearch/plugins/PluginManager.java index 410bf1a8411..fa190354018 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginManager.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginManager.java @@ -82,6 +82,7 @@ public class PluginManager { "lang-groovy", "lang-javascript", "lang-python", + "mapper-attachments", "mapper-murmur3", "mapper-size", "repository-azure", diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index e9bbe6e8114..b9663e4a0a0 100644 --- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -800,7 +800,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ currentFieldName = parser.currentName(); token = parser.nextToken(); if (token == XContentParser.Token.START_OBJECT) { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(parser.contentType()); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder(); xContentBuilder.startObject(); xContentBuilder.field(currentFieldName); xContentBuilder.copyCurrentStructure(parser); @@ -813,22 +813,21 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ } builder.aggregations = aggregations; } else if (context.parseFieldMatcher().match(currentFieldName, HIGHLIGHT_FIELD)) { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(parser.contentType()).copyCurrentStructure(parser); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); builder.highlightBuilder = xContentBuilder.bytes(); } else if (context.parseFieldMatcher().match(currentFieldName, INNER_HITS_FIELD)) { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(parser.contentType()).copyCurrentStructure(parser); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); builder.innerHitsBuilder = xContentBuilder.bytes(); } else if (context.parseFieldMatcher().match(currentFieldName, SUGGEST_FIELD)) { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(parser.contentType()); - xContentBuilder.copyCurrentStructure(parser); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); builder.suggestBuilder = xContentBuilder.bytes(); } else if (context.parseFieldMatcher().match(currentFieldName, SORT_FIELD)) { List sorts = new ArrayList<>(); - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(parser.contentType()).copyCurrentStructure(parser); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); sorts.add(xContentBuilder.bytes()); builder.sorts = sorts; } else if (context.parseFieldMatcher().match(currentFieldName, EXT_FIELD)) { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(parser.contentType()).copyCurrentStructure(parser); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); builder.ext = xContentBuilder.bytes(); } else { throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", @@ -861,14 +860,14 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ } else if (context.parseFieldMatcher().match(currentFieldName, SORT_FIELD)) { List sorts = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(parser.contentType()).copyCurrentStructure(parser); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); sorts.add(xContentBuilder.bytes()); } builder.sorts = sorts; } else if (context.parseFieldMatcher().match(currentFieldName, RESCORE_FIELD)) { List rescoreBuilders = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(parser.contentType()).copyCurrentStructure(parser); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); rescoreBuilders.add(xContentBuilder.bytes()); } builder.rescoreBuilders = rescoreBuilders; diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java index e69ca5a3fce..1174fcdd8a9 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java @@ -41,6 +41,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.object.ObjectMapper; +import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; @@ -196,14 +197,16 @@ public class DefaultSearchContext extends SearchContext { if (query() == null) { parsedQuery(ParsedQuery.parsedMatchAllQuery()); } - if (queryBoost() != 1.0f) { + if (queryBoost() != AbstractQueryBuilder.DEFAULT_BOOST) { parsedQuery(new ParsedQuery(new FunctionScoreQuery(query(), new WeightFactorFunction(queryBoost)), parsedQuery())); } Query searchFilter = searchFilter(types()); if (searchFilter != null) { if (Queries.isConstantMatchAllQuery(query())) { Query q = new ConstantScoreQuery(searchFilter); - q.setBoost(query().getBoost()); + if (query().getBoost() != AbstractQueryBuilder.DEFAULT_BOOST) { + q = new BoostQuery(q, query().getBoost()); + } parsedQuery(new ParsedQuery(q, parsedQuery())); } else { BooleanQuery filtered = new BooleanQuery.Builder() diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy index 5ced769c3e5..ef194768e23 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -38,7 +38,7 @@ grant codeBase "${codebase.securesm-1.0.jar}" { //// Very special jar permissions: //// These are dangerous permissions that we don't want to grant to everything. -grant codeBase "${codebase.lucene-core-5.4.0-snapshot-1711508.jar}" { +grant codeBase "${codebase.lucene-core-5.4.0-snapshot-1712973.jar}" { // needed to allow MMapDirectory's "unmap hack" permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy index e1494357030..b879a0448de 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy @@ -30,7 +30,7 @@ grant codeBase "${codebase.securemock-1.1.jar}" { permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; -grant codeBase "${codebase.lucene-test-framework-5.4.0-snapshot-1711508.jar}" { +grant codeBase "${codebase.lucene-test-framework-5.4.0-snapshot-1712973.jar}" { // needed by RamUsageTester permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; diff --git a/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help b/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help index 7486d98bcb6..389f6d9caa1 100644 --- a/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help +++ b/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help @@ -47,6 +47,7 @@ OFFICIAL PLUGINS - lang-groovy - lang-javascript - lang-python + - mapper-attachments - mapper-murmur3 - mapper-size - repository-azure diff --git a/core/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java b/core/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java new file mode 100644 index 00000000000..fdb482774c6 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java @@ -0,0 +1,116 @@ +package org.elasticsearch.action.support.master; + +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.discovery.DiscoverySettings; +import org.elasticsearch.discovery.zen.elect.ElectMasterService; +import org.elasticsearch.discovery.zen.fd.FaultDetection; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.disruption.NetworkDisconnectPartition; +import org.elasticsearch.test.disruption.NetworkPartition; +import org.elasticsearch.test.transport.MockTransportService; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; + +import static org.hamcrest.Matchers.equalTo; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +@ESIntegTestCase.SuppressLocalMode +public class IndexingMasterFailoverIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + final HashSet> classes = new HashSet<>(super.nodePlugins()); + classes.add(MockTransportService.TestPlugin.class); + return classes; + } + + /** + * Indexing operations which entail mapping changes require a blocking request to the master node to update the mapping. + * If the master node is being disrupted or if it cannot commit cluster state changes, it needs to retry within timeout limits. + * This retry logic is implemented in TransportMasterNodeAction and tested by the following master failover scenario. + */ + public void testMasterFailoverDuringIndexingWithMappingChanges() throws Throwable { + logger.info("--> start 4 nodes, 3 master, 1 data"); + + final Settings sharedSettings = Settings.builder() + .put(FaultDetection.SETTING_PING_TIMEOUT, "1s") // for hitting simulated network failures quickly + .put(FaultDetection.SETTING_PING_RETRIES, "1") // for hitting simulated network failures quickly + .put("discovery.zen.join_timeout", "10s") // still long to induce failures but to long so test won't time out + .put(DiscoverySettings.PUBLISH_TIMEOUT, "1s") // <-- for hitting simulated network failures quickly + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES, 2) + .put("transport.host", "127.0.0.1") // only bind on one IF we use v4 here by default + .build(); + + internalCluster().startMasterOnlyNodesAsync(3, sharedSettings).get(); + + String dataNode = internalCluster().startDataOnlyNode(sharedSettings); + + logger.info("--> wait for all nodes to join the cluster"); + ensureStableCluster(4); + + // We index data with mapping changes into cluster and have master failover at same time + client().admin().indices().prepareCreate("myindex") + .setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)) + .get(); + ensureGreen("myindex"); + + final CyclicBarrier barrier = new CyclicBarrier(2); + + Thread indexingThread = new Thread(new Runnable() { + @Override + public void run() { + try { + barrier.await(); + } catch (InterruptedException e) { + logger.warn("Barrier interrupted", e); + return; + } catch (BrokenBarrierException e) { + logger.warn("Broken barrier", e); + return; + } + for (int i = 0; i < 10; i++) { + // index data with mapping changes + IndexResponse response = client(dataNode).prepareIndex("myindex", "mytype").setSource("field_" + i, "val").get(); + assertThat(response.isCreated(), equalTo(true)); + } + } + }); + indexingThread.setName("indexingThread"); + indexingThread.start(); + + barrier.await(); + + // interrupt communication between master and other nodes in cluster + String master = internalCluster().getMasterName(); + Set otherNodes = new HashSet<>(Arrays.asList(internalCluster().getNodeNames())); + otherNodes.remove(master); + + NetworkPartition partition = new NetworkDisconnectPartition(Collections.singleton(master), otherNodes, random()); + internalCluster().setDisruptionScheme(partition); + + logger.info("--> disrupting network"); + partition.startDisrupting(); + + logger.info("--> waiting for new master to be elected"); + ensureStableCluster(3, dataNode); + + partition.stopDisrupting(); + logger.info("--> waiting to heal"); + ensureStableCluster(4); + + indexingThread.join(); + + ensureGreen("myindex"); + refresh(); + assertThat(client().prepareSearch("myindex").get().getHits().getTotalHits(), equalTo(10L)); + } +} diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java index 59a69149bed..74b78840fd4 100644 --- a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.action.support.ThreadedActionListener; import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -38,6 +39,8 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; @@ -256,17 +259,8 @@ public class TransportMasterNodeActionTests extends ESTestCase { clusterService.setState(ClusterStateCreationUtils.state(localNode, remoteNode, allNodes)); PlainActionFuture listener = new PlainActionFuture<>(); - final AtomicBoolean delegationToMaster = new AtomicBoolean(); + new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool).execute(request, listener); - new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool) { - @Override - protected void processBeforeDelegationToMaster(Request request, ClusterState state) { - logger.debug("Delegation to master called"); - delegationToMaster.set(true); - } - }.execute(request, listener); - - assertTrue("processBeforeDelegationToMaster not called", delegationToMaster.get()); assertThat(transport.capturedRequests().length, equalTo(1)); CapturingTransport.CapturedRequest capturedRequest = transport.capturedRequests()[0]; assertTrue(capturedRequest.node.isMasterNode()); @@ -285,17 +279,8 @@ public class TransportMasterNodeActionTests extends ESTestCase { clusterService.setState(ClusterStateCreationUtils.state(localNode, remoteNode, allNodes)); PlainActionFuture listener = new PlainActionFuture<>(); - final AtomicBoolean delegationToMaster = new AtomicBoolean(); + new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool).execute(request, listener); - new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool) { - @Override - protected void processBeforeDelegationToMaster(Request request, ClusterState state) { - logger.debug("Delegation to master called"); - delegationToMaster.set(true); - } - }.execute(request, listener); - - assertTrue("processBeforeDelegationToMaster not called", delegationToMaster.get()); assertThat(transport.capturedRequests().length, equalTo(1)); CapturingTransport.CapturedRequest capturedRequest = transport.capturedRequests()[0]; assertTrue(capturedRequest.node.isMasterNode()); @@ -320,4 +305,35 @@ public class TransportMasterNodeActionTests extends ESTestCase { } } } + + public void testMasterFailoverAfterStepDown() throws ExecutionException, InterruptedException { + Request request = new Request().masterNodeTimeout(TimeValue.timeValueHours(1)); + PlainActionFuture listener = new PlainActionFuture<>(); + + final Response response = new Response(); + + clusterService.setState(ClusterStateCreationUtils.state(localNode, localNode, allNodes)); + + new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool) { + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + // The other node has become master, simulate failures of this node while publishing cluster state through ZenDiscovery + TransportMasterNodeActionTests.this.clusterService.setState(ClusterStateCreationUtils.state(localNode, remoteNode, allNodes)); + Throwable failure = randomBoolean() + ? new Discovery.FailedToCommitClusterStateException("Fake error") + : new NotMasterException("Fake error"); + listener.onFailure(failure); + } + }.execute(request, listener); + + assertThat(transport.capturedRequests().length, equalTo(1)); + CapturingTransport.CapturedRequest capturedRequest = transport.capturedRequests()[0]; + assertTrue(capturedRequest.node.isMasterNode()); + assertThat(capturedRequest.request, equalTo(request)); + assertThat(capturedRequest.action, equalTo("testAction")); + + transport.handleResponse(capturedRequest.requestId, response); + assertTrue(listener.isDone()); + assertThat(listener.get(), equalTo(response)); + } } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java index af71688e19a..ff6f1ea783b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java @@ -27,13 +27,16 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; +import org.elasticsearch.test.gateway.NoopGatewayAllocator; -import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; -import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; -import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.elasticsearch.cluster.routing.ShardRoutingState.*; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; @@ -624,4 +627,93 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { assertThat(routingNodes.node("node3").isEmpty(), equalTo(true)); } + + public void testRebalanceWhileShardFetching() { + final AtomicBoolean hasFetches = new AtomicBoolean(true); + AllocationService strategy = createAllocationService(settingsBuilder().put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, + ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString()).build(), new NoopGatewayAllocator() { + @Override + public boolean allocateUnassigned(RoutingAllocation allocation) { + if (hasFetches.get()) { + allocation.setHasPendingAsyncFetch(); + } + return super.allocateUnassigned(allocation); + } + }); + + MetaData metaData = MetaData.builder() + .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0)) + .put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT).put(FilterAllocationDecider.INDEX_ROUTING_EXCLUDE_GROUP + "_id", "node1,node2")).numberOfShards(2).numberOfReplicas(0)) + .build(); + + // we use a second index here (test1) that never gets assigned otherwise allocateUnassinged is never called if we don't have unassigned shards. + RoutingTable routingTable = RoutingTable.builder() + .addAsNew(metaData.index("test")) + .addAsNew(metaData.index("test1")) + .build(); + + ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build(); + + logger.info("start two nodes"); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + for (int i = 0; i < routingTable.index("test").shards().size(); i++) { + assertThat(routingTable.index("test").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test").shard(i).primaryShard().state(), equalTo(INITIALIZING)); + } + + logger.debug("start all the primary shards for test"); + RoutingNodes routingNodes = clusterState.getRoutingNodes(); + routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test", INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + for (int i = 0; i < routingTable.index("test").shards().size(); i++) { + assertThat(routingTable.index("test").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test").shard(i).primaryShard().state(), equalTo(STARTED)); + } + + logger.debug("now, start 1 more node, check that rebalancing will not happen since we have shard sync going on"); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) + .put(newNode("node2"))) + .build(); + logger.debug("reroute and check that nothing has changed"); + RoutingAllocation.Result reroute = strategy.reroute(clusterState); + assertFalse(reroute.changed()); + routingTable = reroute.routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + for (int i = 0; i < routingTable.index("test").shards().size(); i++) { + assertThat(routingTable.index("test").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test").shard(i).primaryShard().state(), equalTo(STARTED)); + } + for (int i = 0; i < routingTable.index("test1").shards().size(); i++) { + assertThat(routingTable.index("test1").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test1").shard(i).primaryShard().state(), equalTo(UNASSIGNED)); + } + logger.debug("now set hasFetches to true and reroute we should now see exactly one relocating shard"); + hasFetches.set(false); + reroute = strategy.reroute(clusterState); + assertTrue(reroute.changed()); + routingTable = reroute.routingTable(); + int numStarted = 0; + int numRelocating = 0; + for (int i = 0; i < routingTable.index("test").shards().size(); i++) { + + assertThat(routingTable.index("test").shard(i).shards().size(), equalTo(1)); + if (routingTable.index("test").shard(i).primaryShard().state() == STARTED) { + numStarted++; + } else if (routingTable.index("test").shard(i).primaryShard().state() == RELOCATING) { + numRelocating++; + } + } + for (int i = 0; i < routingTable.index("test1").shards().size(); i++) { + assertThat(routingTable.index("test1").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test1").shard(i).primaryShard().state(), equalTo(UNASSIGNED)); + } + assertEquals(numStarted, 1); + assertEquals(numRelocating, 1); + + } } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java index 9ecae9c6760..e9d6143b6cd 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESAllocationTestCase; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; @@ -87,6 +88,64 @@ public class ShardsLimitAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); } + public void testClusterLevelShardsLimitAllocate() { + AllocationService strategy = createAllocationService(settingsBuilder() + .put("cluster.routing.allocation.concurrent_recoveries", 10) + .put(ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE, 1) + .build()); + + logger.info("Building initial routing table"); + + MetaData metaData = MetaData.builder() + .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))) + .build(); + + RoutingTable routingTable = RoutingTable.builder() + .addAsNew(metaData.index("test")) + .build(); + + ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build(); + logger.info("Adding two nodes and performing rerouting"); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1)); + assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1)); + + logger.info("Start the primary shards"); + RoutingNodes routingNodes = clusterState.getRoutingNodes(); + routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(1)); + assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(1)); + assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(2)); + + // Bump the cluster total shards to 2 + strategy = createAllocationService(settingsBuilder() + .put("cluster.routing.allocation.concurrent_recoveries", 10) + .put(ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE, 2) + .build()); + + logger.info("Do another reroute, make sure shards are now allocated"); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1)); + assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1)); + + routingNodes = clusterState.getRoutingNodes(); + routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(2)); + assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(2)); + assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(0)); + } + public void testIndexLevelShardsLimitRemain() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) diff --git a/core/src/test/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQueryTests.java b/core/src/test/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQueryTests.java index 5cb80607115..9098289847e 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQueryTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQueryTests.java @@ -27,6 +27,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.store.RAMDirectory; @@ -34,6 +35,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.test.ESTestCase; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; public class MultiPhrasePrefixQueryTests extends ESTestCase { public void testSimple() throws Exception { @@ -78,6 +80,8 @@ public class MultiPhrasePrefixQueryTests extends ESTestCase { multiPhrasePrefixQuery.add(new Term[]{new Term("field", "aaa"), new Term("field", "bb")}); multiPhrasePrefixQuery.setBoost(randomFloat()); Query query = multiPhrasePrefixQuery.rewrite(reader); - assertThat(query.getBoost(), equalTo(multiPhrasePrefixQuery.getBoost())); + assertThat(query, instanceOf(BoostQuery.class)); + BoostQuery boostQuery = (BoostQuery) query; + assertThat(boostQuery.getBoost(), equalTo(multiPhrasePrefixQuery.getBoost())); } } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java index e4089b37fce..fda82104de8 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingService; diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 76852621e76..f9ea8579329 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -385,7 +385,7 @@ public class IndexModuleTests extends ESTestCase { } @Override - public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { return null; } } diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index eeb3b41dabe..a8acdfa13f8 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -519,7 +519,7 @@ public class InternalEngineTests extends ESTestCase { } @Override - public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { counter.incrementAndGet(); return searcher; } @@ -530,7 +530,7 @@ public class InternalEngineTests extends ESTestCase { engine.close(); engine = new InternalEngine(engine.config(), false); - Engine.Searcher searcher = wrapper.wrap(engine.config(), engine.acquireSearcher("test")); + Engine.Searcher searcher = wrapper.wrap(engine.acquireSearcher("test")); assertThat(counter.get(), equalTo(2)); searcher.close(); IOUtils.close(store, engine); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 996215cd5b6..df169c27ade 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -37,7 +37,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper.BuilderContext; import org.elasticsearch.index.mapper.MapperBuilders; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.geo.BaseGeoPointFieldMapper; import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; import org.elasticsearch.index.mapper.geo.GeoPointFieldMapperLegacy; import org.elasticsearch.index.mapper.internal.ParentFieldMapper; @@ -95,7 +94,6 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase { } else if (type.getType().equals("byte")) { fieldType = MapperBuilders.byteField(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType(); } else if (type.getType().equals("geo_point")) { - BaseGeoPointFieldMapper.Builder builder; // norelease update to .before(Version.V_2_2_0 once GeoPointFieldV2 is fully merged if (indexService.getIndexSettings().getIndexVersionCreated().onOrBefore(Version.CURRENT)) { fieldType = new GeoPointFieldMapperLegacy.Builder(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType(); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java new file mode 100644 index 00000000000..1e8428e0f8c --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.GeoPointField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.util.GeoUtils; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.VersionUtils; + +import static org.elasticsearch.test.geo.RandomShapeGenerator.randomPoint; +import static org.hamcrest.Matchers.*; + +/** + * + */ +public abstract class AbstractGeoFieldDataTestCase extends AbstractFieldDataImplTestCase { + protected Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + + @Override + protected abstract FieldDataType getFieldDataType(); + + protected Settings.Builder getFieldDataSettings() { + return Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version); + } + + protected Field randomGeoPointField(String fieldName, Field.Store store) { + GeoPoint point = randomPoint(random()); + // norelease move to .before(Version.2_2_0) once GeoPointV2 is fully merged + if (version.onOrBefore(Version.CURRENT)) { + return new StringField(fieldName, point.lat()+","+point.lon(), store); + } + return new GeoPointField(fieldName, point.lon(), point.lat(), store); + } + + @Override + protected void fillAllMissing() throws Exception { + Document d = new Document(); + d.add(new StringField("_id", "1", Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "2", Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "3", Field.Store.NO)); + writer.addDocument(d); + } + + @Override + public void testSortMultiValuesFields() { + assumeFalse("Only test on non geo_point fields", getFieldDataType().equals("geo_point")); + } + + protected void assertValues(MultiGeoPointValues values, int docId) { + assertValues(values, docId, false); + } + + protected void assertMissing(MultiGeoPointValues values, int docId) { + assertValues(values, docId, true); + } + + private void assertValues(MultiGeoPointValues values, int docId, boolean missing) { + values.setDocument(docId); + int docCount = values.count(); + if (missing) { + assertThat(docCount, equalTo(0)); + } else { + assertThat(docCount, greaterThan(0)); + for (int i = 0; i < docCount; ++i) { + assertThat(values.valueAt(i).lat(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LAT_INCL), lessThanOrEqualTo(GeoUtils.MAX_LAT_INCL))); + assertThat(values.valueAt(i).lat(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LON_INCL), lessThanOrEqualTo(GeoUtils.MAX_LON_INCL))); + } + } + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java new file mode 100644 index 00000000000..cad968b837c --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java @@ -0,0 +1,205 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.document.*; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.elasticsearch.index.fielddata.plain.AbstractAtomicGeoPointFieldData; + +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +/** + * Basic Unit Test for GeoPointField data + * todo include backcompat testing - see ISSUE #14562 + */ +public class GeoFieldDataTests extends AbstractGeoFieldDataTestCase { + private static String FIELD_NAME = "value"; + + @Override + protected FieldDataType getFieldDataType() { + return new FieldDataType("geo_point", getFieldDataSettings()); + } + + @Override + protected void add2SingleValuedDocumentsAndDeleteOneOfThem() throws Exception { + Document d = new Document(); + + d.add(new StringField("_id", "1", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.YES)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "2", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + + writer.commit(); + + writer.deleteDocuments(new Term("_id", "1")); + } + + @Override + protected void fillMultiValueWithMissing() throws Exception { + Document d = new Document(); + d.add(new StringField("_id", "1", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + + // missing + d = new Document(); + d.add(new StringField("_id", "2", Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "3", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + } + + @Override + protected void fillSingleValueAllSet() throws Exception { + Document d = new Document(); + d.add(new StringField("_id", "1", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "2", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "3", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + } + + @Override + protected void fillSingleValueWithMissing() throws Exception { + Document d = new Document(); + d.add(new StringField("_id", "1", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "2", Field.Store.NO)); + //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING.... + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "3", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + } + + @Override + protected void fillMultiValueAllSet() throws Exception { + Document d = new Document(); + d.add(new StringField("_id", "1", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "2", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "3", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + } + + @Override + protected void fillExtendedMvSet() throws Exception { + Document d; + final int maxDocs = randomInt(10); + for (int i=0; i> extends ESTestCase { @@ -336,13 +336,24 @@ public abstract class AbstractQueryTestCase> public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTQUERIES; runs++) { QB testQuery = createTestQueryBuilder(); - assertParsedQuery(testQuery.toString(), testQuery); + XContentBuilder builder = toXContent(testQuery, randomFrom(XContentType.values())); + assertParsedQuery(builder.bytes(), testQuery); for (Map.Entry alternateVersion : getAlternateVersions().entrySet()) { - assertParsedQuery(alternateVersion.getKey(), alternateVersion.getValue()); + String queryAsString = alternateVersion.getKey(); + assertParsedQuery(new BytesArray(queryAsString), alternateVersion.getValue()); } } } + protected static XContentBuilder toXContent(QueryBuilder query, XContentType contentType) throws IOException { + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + if (randomBoolean()) { + builder.prettyPrint(); + } + query.toXContent(builder, ToXContent.EMPTY_PARAMS); + return builder; + } + /** * Test that unknown field trigger ParsingException. * To find the right position in the root query, we add a marker as `queryName` which @@ -356,7 +367,7 @@ public abstract class AbstractQueryTestCase> testQuery = createTestQueryBuilder(); } while (testQuery.toString().contains(marker)); testQuery.queryName(marker); // to find root query to add additional bogus field there - String queryAsString = testQuery.toString().replace("\"" + marker + "\"", "\"" + marker + "\", \"bogusField\" : \"someValue\""); + String queryAsString = testQuery.toString().replace("\"" + marker + "\"", "\"" + marker + "\", \"bogusField\" : \"someValue\""); try { parseQuery(queryAsString); fail("ParsingException expected."); @@ -411,6 +422,20 @@ public abstract class AbstractQueryTestCase> assertEquals(expectedQuery.hashCode(), newQuery.hashCode()); } + /** + * Parses the query provided as bytes argument and compares it with the expected result provided as argument as a {@link QueryBuilder} + */ + protected final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery) throws IOException { + assertParsedQuery(queryAsBytes, expectedQuery, ParseFieldMatcher.STRICT); + } + + protected final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException { + QueryBuilder newQuery = parseQuery(queryAsBytes, matcher); + assertNotSame(newQuery, expectedQuery); + assertEquals(expectedQuery, newQuery); + assertEquals(expectedQuery.hashCode(), newQuery.hashCode()); + } + protected final QueryBuilder parseQuery(String queryAsString) throws IOException { return parseQuery(queryAsString, ParseFieldMatcher.STRICT); } @@ -420,12 +445,16 @@ public abstract class AbstractQueryTestCase> return parseQuery(parser, matcher); } - protected final QueryBuilder parseQuery(BytesReference query) throws IOException { - XContentParser parser = XContentFactory.xContent(query).createParser(query); - return parseQuery(parser, ParseFieldMatcher.STRICT); + protected final QueryBuilder parseQuery(BytesReference queryAsBytes) throws IOException { + return parseQuery(queryAsBytes, ParseFieldMatcher.STRICT); } - protected final QueryBuilder parseQuery(XContentParser parser, ParseFieldMatcher matcher) throws IOException { + protected final QueryBuilder parseQuery(BytesReference queryAsBytes, ParseFieldMatcher matcher) throws IOException { + XContentParser parser = XContentFactory.xContent(queryAsBytes).createParser(queryAsBytes); + return parseQuery(parser, matcher); + } + + private QueryBuilder parseQuery(XContentParser parser, ParseFieldMatcher matcher) throws IOException { QueryParseContext context = createParseContext(); context.reset(parser); context.parseFieldMatcher(matcher); @@ -501,26 +530,45 @@ public abstract class AbstractQueryTestCase> assertThat(namedQuery, equalTo(query)); } if (query != null) { - assertBoost(queryBuilder, query); + if (queryBuilder.boost() != AbstractQueryBuilder.DEFAULT_BOOST) { + assertThat(query, either(instanceOf(BoostQuery.class)).or(instanceOf(SpanBoostQuery.class))); + if (query instanceof SpanBoostQuery) { + SpanBoostQuery spanBoostQuery = (SpanBoostQuery) query; + assertThat(spanBoostQuery.getBoost(), equalTo(queryBuilder.boost())); + query = spanBoostQuery.getQuery(); + } else { + BoostQuery boostQuery = (BoostQuery) query; + assertThat(boostQuery.getBoost(), equalTo(queryBuilder.boost())); + query = boostQuery.getQuery(); + } + } } doAssertLuceneQuery(queryBuilder, query, context); } - /** - * Allows to override boost assertions for queries that don't have the default behaviour - */ - protected void assertBoost(QB queryBuilder, Query query) throws IOException { - // workaround https://bugs.openjdk.java.net/browse/JDK-8056984 - float boost = queryBuilder.boost(); - assertThat(query.getBoost(), equalTo(boost)); - } - /** * Checks the result of {@link QueryBuilder#toQuery(QueryShardContext)} given the original {@link QueryBuilder} and {@link QueryShardContext}. * Contains the query specific checks to be implemented by subclasses. */ protected abstract void doAssertLuceneQuery(QB queryBuilder, Query query, QueryShardContext context) throws IOException; + protected static void assertTermOrBoostQuery(Query query, String field, String value, float fieldBoost) { + if (fieldBoost != AbstractQueryBuilder.DEFAULT_BOOST) { + assertThat(query, instanceOf(BoostQuery.class)); + BoostQuery boostQuery = (BoostQuery) query; + assertThat(boostQuery.getBoost(), equalTo(fieldBoost)); + query = boostQuery.getQuery(); + } + assertTermQuery(query, field, value); + } + + protected static void assertTermQuery(Query query, String field, String value) { + assertThat(query, instanceOf(TermQuery.class)); + TermQuery termQuery = (TermQuery) query; + assertThat(termQuery.getTerm().field(), equalTo(field)); + assertThat(termQuery.getTerm().text().toLowerCase(Locale.ROOT), equalTo(value.toLowerCase(Locale.ROOT))); + } + /** * Test serialization and deserialization of the test query. */ diff --git a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java index 04f07a9428c..7bd8ffeefd3 100644 --- a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java @@ -20,12 +20,15 @@ package org.elasticsearch.index.query; import org.apache.lucene.search.*; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.hamcrest.Matchers; import java.io.IOException; import java.util.*; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.*; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; @@ -174,7 +177,8 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase { /** @@ -138,9 +133,13 @@ public class DisMaxQueryBuilderTests extends AbstractQueryTestCase disjuncts = disjunctionMaxQuery.getDisjuncts(); assertThat(disjuncts.size(), equalTo(1)); - PrefixQuery firstQ = (PrefixQuery) disjuncts.get(0); + assertThat(disjuncts.get(0), instanceOf(BoostQuery.class)); + BoostQuery boostQuery = (BoostQuery) disjuncts.get(0); + assertThat((double) boostQuery.getBoost(), closeTo(1.2, 0.00001)); + assertThat(boostQuery.getQuery(), instanceOf(PrefixQuery.class)); + PrefixQuery firstQ = (PrefixQuery) boostQuery.getQuery(); // since age is automatically registered in data, we encode it as numeric assertThat(firstQ.getPrefix(), equalTo(new Term(STRING_FIELD_NAME, "sh"))); - assertThat((double) firstQ.getBoost(), closeTo(1.2, 0.00001)); + } } diff --git a/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java index c7787860c01..f1511bc3909 100644 --- a/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.Term; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.Query; @@ -116,12 +117,15 @@ public class FuzzyQueryBuilderTests extends AbstractQueryTestCase 0); } - Object value = ""; + Object value; if (fieldName.equals(STRING_FIELD_NAME)) { int terms = randomIntBetween(0, 3); StringBuilder builder = new StringBuilder(); for (int i = 0; i < terms; i++) { - builder.append(randomAsciiOfLengthBetween(1, 10) + " "); + builder.append(randomAsciiOfLengthBetween(1, 10)).append(" "); } value = builder.toString().trim(); } else { @@ -139,8 +133,6 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase 0); QueryShardContext shardContext = createShardContext(); MultiMatchQueryBuilder multiMatchQueryBuilder = new MultiMatchQueryBuilder("test"); - multiMatchQueryBuilder.field(STRING_FIELD_NAME, 5); + multiMatchQueryBuilder.field(STRING_FIELD_NAME, 5f); Query query = multiMatchQueryBuilder.toQuery(shardContext); - assertThat(query, instanceOf(TermQuery.class)); - assertThat(query.getBoost(), equalTo(5f)); + assertTermOrBoostQuery(query, STRING_FIELD_NAME, "test", 5f); multiMatchQueryBuilder = new MultiMatchQueryBuilder("test"); - multiMatchQueryBuilder.field(STRING_FIELD_NAME, 5); - multiMatchQueryBuilder.boost(2); + multiMatchQueryBuilder.field(STRING_FIELD_NAME, 5f); + multiMatchQueryBuilder.boost(2f); query = multiMatchQueryBuilder.toQuery(shardContext); - assertThat(query, instanceOf(TermQuery.class)); - assertThat(query.getBoost(), equalTo(10f)); + assertThat(query, instanceOf(BoostQuery.class)); + BoostQuery boostQuery = (BoostQuery) query; + assertThat(boostQuery.getBoost(), equalTo(2f)); + assertTermOrBoostQuery(boostQuery.getQuery(), STRING_FIELD_NAME, "test", 5f); } public void testToQueryMultipleTermsBooleanQuery() throws Exception { @@ -212,7 +201,9 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase disjuncts = disMaxQuery.getDisjuncts(); + assertThat(disjuncts.get(0), instanceOf(TermQuery.class)); assertThat(((TermQuery) disjuncts.get(0)).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test"))); + assertThat(disjuncts.get(1), instanceOf(TermQuery.class)); assertThat(((TermQuery) disjuncts.get(1)).getTerm(), equalTo(new Term(STRING_FIELD_NAME_2, "test"))); } diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index 00b139039cf..2f2fdd406e1 100644 --- a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -20,16 +20,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.BoostQuery; -import org.apache.lucene.search.DisjunctionMaxQuery; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.MatchNoDocsQuery; -import org.apache.lucene.search.NumericRangeQuery; -import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.RegexpQuery; -import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.*; import org.apache.lucene.util.automaton.TooComplexToDeterminizeException; import org.elasticsearch.common.lucene.all.AllTermQuery; import org.hamcrest.Matchers; @@ -42,9 +33,8 @@ import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBooleanSubQuery; import static org.hamcrest.CoreMatchers.either; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.instanceOf; public class QueryStringQueryBuilderTests extends AbstractQueryTestCase { @@ -197,13 +187,17 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase disjuncts = disMaxQuery.getDisjuncts(); - assertThat(((TermQuery) disjuncts.get(0)).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test"))); - assertThat((double) disjuncts.get(0).getBoost(), closeTo(2.2, 0.01)); - assertThat(((TermQuery) disjuncts.get(1)).getTerm(), equalTo(new Term(STRING_FIELD_NAME_2, "test"))); - assertThat((double) disjuncts.get(1).getBoost(), closeTo(1, 0.01)); + assertTermOrBoostQuery(disjuncts.get(0), STRING_FIELD_NAME, "test", 2.2f); + assertTermOrBoostQuery(disjuncts.get(1), STRING_FIELD_NAME_2, "test", 1.0f); } public void testToQueryRegExpQuery() throws Exception { @@ -330,4 +322,56 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase 0); + int numBoosts = randomIntBetween(2, 10); + float[] boosts = new float[numBoosts + 1]; + String queryStringPrefix = ""; + String queryStringSuffix = ""; + for (int i = 0; i < boosts.length - 1; i++) { + float boost = 2.0f / randomIntBetween(3, 20); + boosts[i] = boost; + queryStringPrefix += "("; + queryStringSuffix += ")^" + boost; + } + String queryString = queryStringPrefix + "foo bar" + queryStringSuffix; + + float mainBoost = 2.0f / randomIntBetween(3, 20); + boosts[boosts.length - 1] = mainBoost; + QueryStringQueryBuilder queryStringQueryBuilder = new QueryStringQueryBuilder(queryString).field(STRING_FIELD_NAME) + .minimumShouldMatch("2").boost(mainBoost); + Query query = queryStringQueryBuilder.toQuery(createShardContext()); + + for (int i = boosts.length - 1; i >= 0; i--) { + assertThat(query, instanceOf(BoostQuery.class)); + BoostQuery boostQuery = (BoostQuery) query; + assertThat(boostQuery.getBoost(), equalTo(boosts[i])); + query = boostQuery.getQuery(); + } + + assertThat(query, instanceOf(BooleanQuery.class)); + BooleanQuery booleanQuery = (BooleanQuery) query; + assertThat(booleanQuery.getMinimumNumberShouldMatch(), equalTo(2)); + assertThat(booleanQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.SHOULD)); + assertThat(booleanQuery.clauses().get(0).getQuery(), equalTo(new TermQuery(new Term(STRING_FIELD_NAME, "foo")))); + assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.SHOULD)); + assertThat(booleanQuery.clauses().get(1).getQuery(), equalTo(new TermQuery(new Term(STRING_FIELD_NAME, "bar")))); + } + + public void testToQueryPhraseQueryBoostAndSlop() throws IOException { + assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0); + QueryStringQueryBuilder queryStringQueryBuilder = new QueryStringQueryBuilder("\"test phrase\"~2").field(STRING_FIELD_NAME, 5f); + Query query = queryStringQueryBuilder.toQuery(createShardContext()); + assertThat(query, instanceOf(DisjunctionMaxQuery.class)); + DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query; + assertThat(disjunctionMaxQuery.getDisjuncts().size(), equalTo(1)); + assertThat(disjunctionMaxQuery.getDisjuncts().get(0), instanceOf(BoostQuery.class)); + BoostQuery boostQuery = (BoostQuery) disjunctionMaxQuery.getDisjuncts().get(0); + assertThat(boostQuery.getBoost(), equalTo(5f)); + assertThat(boostQuery.getQuery(), instanceOf(PhraseQuery.class)); + PhraseQuery phraseQuery = (PhraseQuery) boostQuery.getQuery(); + assertThat(phraseQuery.getSlop(), Matchers.equalTo(2)); + assertThat(phraseQuery.getTerms().length, equalTo(2)); + } } diff --git a/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java b/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java index 2b173bdb1fe..147d21576c6 100644 --- a/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java +++ b/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java @@ -61,27 +61,38 @@ public class RandomQueryBuilder { public static MultiTermQueryBuilder createMultiTermQuery(Random r) { // for now, only use String Rangequeries for MultiTerm test, numeric and date makes little sense // see issue #12123 for discussion + MultiTermQueryBuilder multiTermQueryBuilder; switch(RandomInts.randomIntBetween(r, 0, 5)) { case 0: RangeQueryBuilder stringRangeQuery = new RangeQueryBuilder(AbstractQueryTestCase.STRING_FIELD_NAME); stringRangeQuery.from("a" + RandomStrings.randomAsciiOfLengthBetween(r, 1, 10)); stringRangeQuery.to("z" + RandomStrings.randomAsciiOfLengthBetween(r, 1, 10)); - return stringRangeQuery; + multiTermQueryBuilder = stringRangeQuery; + break; case 1: RangeQueryBuilder numericRangeQuery = new RangeQueryBuilder(AbstractQueryTestCase.INT_FIELD_NAME); numericRangeQuery.from(RandomInts.randomIntBetween(r, 1, 100)); numericRangeQuery.to(RandomInts.randomIntBetween(r, 101, 200)); - return numericRangeQuery; + multiTermQueryBuilder = numericRangeQuery; + break; case 2: - return new FuzzyQueryBuilder(AbstractQueryTestCase.INT_FIELD_NAME, RandomInts.randomInt(r, 1000)); + multiTermQueryBuilder = new FuzzyQueryBuilder(AbstractQueryTestCase.INT_FIELD_NAME, RandomInts.randomInt(r, 1000)); + break; case 3: - return new FuzzyQueryBuilder(AbstractQueryTestCase.STRING_FIELD_NAME, RandomStrings.randomAsciiOfLengthBetween(r, 1, 10)); + multiTermQueryBuilder = new FuzzyQueryBuilder(AbstractQueryTestCase.STRING_FIELD_NAME, RandomStrings.randomAsciiOfLengthBetween(r, 1, 10)); + break; case 4: - return new PrefixQueryBuilderTests().createTestQueryBuilder(); + multiTermQueryBuilder = new PrefixQueryBuilderTests().createTestQueryBuilder(); + break; case 5: - return new WildcardQueryBuilderTests().createTestQueryBuilder(); + multiTermQueryBuilder = new WildcardQueryBuilderTests().createTestQueryBuilder(); + break; default: throw new UnsupportedOperationException(); } + if (r.nextBoolean()) { + multiTermQueryBuilder.boost(2.0f / RandomInts.randomIntBetween(r, 1, 20)); + } + return multiTermQueryBuilder; } } diff --git a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java index 85f1c1d5f0a..02a85a28f0b 100644 --- a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java @@ -20,17 +20,10 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.MatchNoDocsQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.*; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.HashMap; @@ -239,16 +232,13 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase 1) { - assertTrue("Query should have been BooleanQuery but was " + query.getClass().getName(), query instanceof BooleanQuery); - + assertThat(query, instanceOf(BooleanQuery.class)); BooleanQuery boolQuery = (BooleanQuery) query; if (queryBuilder.lowercaseExpandedTerms()) { for (BooleanClause clause : boolQuery.clauses()) { @@ -289,43 +278,26 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase fields = queryBuilder.fields().keySet().iterator(); + Iterator> fieldsIterator = queryBuilder.fields().entrySet().iterator(); for (BooleanClause booleanClause : boolQuery) { - assertThat(booleanClause.getQuery(), instanceOf(TermQuery.class)); - TermQuery termQuery = (TermQuery) booleanClause.getQuery(); - assertThat(termQuery.getTerm().field(), equalTo(fields.next())); - assertThat(termQuery.getTerm().text().toLowerCase(Locale.ROOT), equalTo(queryBuilder.value().toLowerCase(Locale.ROOT))); + Map.Entry field = fieldsIterator.next(); + assertTermOrBoostQuery(booleanClause.getQuery(), field.getKey(), queryBuilder.value(), field.getValue()); } - if (queryBuilder.minimumShouldMatch() != null) { assertThat(boolQuery.getMinimumNumberShouldMatch(), greaterThan(0)); } - } else if (queryBuilder.fields().size() <= 1) { - assertTrue("Query should have been TermQuery but was " + query.getClass().getName(), query instanceof TermQuery); - - TermQuery termQuery = (TermQuery) query; - String field; - if (queryBuilder.fields().size() == 0) { - field = MetaData.ALL; - } else { - field = queryBuilder.fields().keySet().iterator().next(); - } - assertThat(termQuery.getTerm().field(), equalTo(field)); - assertThat(termQuery.getTerm().text().toLowerCase(Locale.ROOT), equalTo(queryBuilder.value().toLowerCase(Locale.ROOT))); + } else if (queryBuilder.fields().size() == 1) { + Map.Entry field = queryBuilder.fields().entrySet().iterator().next(); + assertTermOrBoostQuery(query, field.getKey(), queryBuilder.value(), field.getValue()); + } else if (queryBuilder.fields().size() == 0) { + assertTermQuery(query, MetaData.ALL, queryBuilder.value()); } else { fail("Encountered lucene query type we do not have a validation implementation for in our " + SimpleQueryStringBuilderTests.class.getSimpleName()); } } - @Override - protected void assertBoost(SimpleQueryStringBuilder queryBuilder, Query query) throws IOException { - //boost may get parsed from the random query, we then combine the main boost with that one coming from lucene - //instead of trying to reparse the query and guess what the boost should be, we delegate boost checks to specific boost tests below - } - - private int shouldClauses(BooleanQuery query) { + private static int shouldClauses(BooleanQuery query) { int result = 0; for (BooleanClause c : query.clauses()) { if (c.getOccur() == BooleanClause.Occur.SHOULD) { @@ -341,15 +313,21 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase((MultiTermQuery)multiTermQuery).getWrappedQuery())); } @@ -72,4 +86,10 @@ public class SpanMultiTermQueryBuilderTests extends AbstractQueryTestCase { if (reader == open) { @@ -118,13 +118,13 @@ public class IndexSearcherWrapperTests extends ESTestCase { } @Override - public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { return searcher; } }; final ConcurrentHashMap cache = new ConcurrentHashMap<>(); try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { - try (final Engine.Searcher wrap = wrapper.wrap(ENGINE_CONFIG, engineSearcher)) { + try (final Engine.Searcher wrap = wrapper.wrap(engineSearcher)) { ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), reader -> { cache.remove(reader.getCoreCacheKey()); }); @@ -154,7 +154,7 @@ public class IndexSearcherWrapperTests extends ESTestCase { searcher.setSimilarity(iwc.getSimilarity()); IndexSearcherWrapper wrapper = new IndexSearcherWrapper(); try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { - final Engine.Searcher wrap = wrapper.wrap(ENGINE_CONFIG, engineSearcher); + final Engine.Searcher wrap = wrapper.wrap(engineSearcher); assertSame(wrap, engineSearcher); } IOUtils.close(open, writer, dir); @@ -180,7 +180,7 @@ public class IndexSearcherWrapperTests extends ESTestCase { }; try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { try { - wrapper.wrap(ENGINE_CONFIG, engineSearcher); + wrapper.wrap(engineSearcher); fail("reader must delegate cache key"); } catch (IllegalStateException ex) { // all is well @@ -194,7 +194,7 @@ public class IndexSearcherWrapperTests extends ESTestCase { }; try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { try { - wrapper.wrap(ENGINE_CONFIG, engineSearcher); + wrapper.wrap(engineSearcher); fail("reader must delegate cache key"); } catch (IllegalStateException ex) { // all is well diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 1c6b80cac79..18dc07d4ef0 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -908,7 +908,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { } @Override - public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { return searcher; } }; @@ -947,7 +947,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { } @Override - public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { return searcher; } }; @@ -990,8 +990,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { throw new RuntimeException("boom"); } - @Override - public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { return searcher; } }; diff --git a/core/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java b/core/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java index cab052a1b0c..8d33758ef26 100644 --- a/core/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java @@ -21,10 +21,16 @@ package org.elasticsearch.recovery; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.Priority; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.zen.ZenDiscovery; +import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; @@ -124,4 +130,36 @@ public class FullRollingRestartIT extends ESIntegTestCase { assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 2000l); } } + + public void testNoRebalanceOnRollingRestart() throws Exception { + // see https://github.com/elastic/elasticsearch/issues/14387 + internalCluster().startMasterOnlyNode(Settings.EMPTY); + internalCluster().startDataOnlyNodesAsync(3).get(); + /** + * We start 3 nodes and a dedicated master. Restart on of the data-nodes and ensure that we got no relocations. + * Yet we have 6 shards 0 replica so that means if the restarting node comes back both other nodes are subject + * to relocating to the restarting node since all had 2 shards and now one node has nothing allocated. + * We have a fix for this to wait until we have allocated unallocated shards now so this shouldn't happen. + */ + prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "6").put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0").put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, TimeValue.timeValueMinutes(1))).get(); + + for (int i = 0; i < 100; i++) { + client().prepareIndex("test", "type1", Long.toString(i)) + .setSource(MapBuilder.newMapBuilder().put("test", "value" + i).map()).execute().actionGet(); + } + ensureGreen(); + ClusterState state = client().admin().cluster().prepareState().get().getState(); + RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").get(); + for (RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) { + assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + recoveryState.getTargetNode() + "\n" + state.prettyPrint(), recoveryState.getType() != RecoveryState.Type.RELOCATION); + } + internalCluster().restartRandomDataNode(); + ensureGreen(); + ClusterState afterState = client().admin().cluster().prepareState().get().getState(); + + recoveryResponse = client().admin().indices().prepareRecoveries("test").get(); + for (RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) { + assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + recoveryState.getTargetNode()+ "-- \nbefore: \n" + state.prettyPrint() + "\nafter: \n" + afterState.prettyPrint(), recoveryState.getType() != RecoveryState.Type.RELOCATION); + } + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/MissingValueIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/MissingValueIT.java index d8be0c2ab35..f2a78295664 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/MissingValueIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/MissingValueIT.java @@ -183,16 +183,20 @@ public class MissingValueIT extends ESIntegTestCase { SearchResponse response = client().prepareSearch("idx").addAggregation(geoBounds("bounds").field("non_existing_field").missing("2,1")).get(); assertSearchResponse(response); GeoBounds bounds = response.getAggregations().get("bounds"); - assertEquals(new GeoPoint(2,1), bounds.bottomRight()); - assertEquals(new GeoPoint(2,1), bounds.topLeft()); + assertThat(bounds.bottomRight().lat(), closeTo(2.0, 1E-5)); + assertThat(bounds.bottomRight().lon(), closeTo(1.0, 1E-5)); + assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5)); + assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5)); } public void testGeoBounds() { SearchResponse response = client().prepareSearch("idx").addAggregation(geoBounds("bounds").field("location").missing("2,1")).get(); assertSearchResponse(response); GeoBounds bounds = response.getAggregations().get("bounds"); - assertEquals(new GeoPoint(1,2), bounds.bottomRight()); - assertEquals(new GeoPoint(2,1), bounds.topLeft()); + assertThat(bounds.bottomRight().lat(), closeTo(1.0, 1E-5)); + assertThat(bounds.bottomRight().lon(), closeTo(2.0, 1E-5)); + assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5)); + assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5)); } public void testGeoCentroid() { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java index e0a3e644c62..c76e3681eb8 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java @@ -18,8 +18,11 @@ */ package org.elasticsearch.search.aggregations.bucket; +import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; @@ -28,6 +31,7 @@ import org.elasticsearch.search.aggregations.bucket.range.Range; import org.elasticsearch.search.aggregations.bucket.range.Range.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.VersionUtils; import org.hamcrest.Matchers; import java.util.ArrayList; @@ -53,6 +57,7 @@ import static org.hamcrest.core.IsNull.nullValue; */ @ESIntegTestCase.SuiteScopeTestCase public class GeoDistanceIT extends ESIntegTestCase { + private Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); private IndexRequestBuilder indexCity(String idx, String name, String... latLons) throws Exception { XContentBuilder source = jsonBuilder().startObject().field("city", name); @@ -67,7 +72,8 @@ public class GeoDistanceIT extends ESIntegTestCase { @Override public void setupSuiteScopeCluster() throws Exception { - prepareCreate("idx") + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + prepareCreate("idx").setSettings(settings) .addMapping("type", "location", "type=geo_point", "city", "type=string,index=not_analyzed") .execute().actionGet(); @@ -109,7 +115,8 @@ public class GeoDistanceIT extends ESIntegTestCase { } } indexRandom(true, cities); - prepareCreate("empty_bucket_idx").addMapping("type", "value", "type=integer", "location", "type=geo_point").execute().actionGet(); + prepareCreate("empty_bucket_idx") + .addMapping("type", "value", "type=integer", "location", "type=geo_point").execute().actionGet(); List builders = new ArrayList<>(); for (int i = 0; i < 2; i++) { builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + i).setSource(jsonBuilder() diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java index 6a49932c4ee..eed080071bb 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java @@ -23,9 +23,12 @@ import com.carrotsearch.hppc.ObjectIntMap; import com.carrotsearch.hppc.cursors.ObjectIntCursor; import org.apache.lucene.util.GeoHashUtils; +import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; @@ -33,6 +36,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid.Bucket; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.VersionUtils; import java.util.ArrayList; import java.util.Arrays; @@ -50,6 +54,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; @ESIntegTestCase.SuiteScopeTestCase public class GeoHashGridIT extends ESIntegTestCase { + private Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); static ObjectIntMap expectedDocCountsForGeoHash = null; static ObjectIntMap multiValuedExpectedDocCountsForGeoHash = null; @@ -74,7 +79,9 @@ public class GeoHashGridIT extends ESIntegTestCase { public void setupSuiteScopeCluster() throws Exception { createIndex("idx_unmapped"); - assertAcked(prepareCreate("idx") + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + + assertAcked(prepareCreate("idx").setSettings(settings) .addMapping("type", "location", "type=geo_point", "city", "type=string,index=not_analyzed")); List cities = new ArrayList<>(); @@ -99,7 +106,7 @@ public class GeoHashGridIT extends ESIntegTestCase { } indexRandom(true, cities); - assertAcked(prepareCreate("multi_valued_idx") + assertAcked(prepareCreate("multi_valued_idx").setSettings(settings) .addMapping("type", "location", "type=geo_point", "city", "type=string,index=not_analyzed")); cities = new ArrayList<>(); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java index 0504cb6ff5e..6419e9dcac3 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java @@ -36,12 +36,7 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.geoBound import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.sameInstance; +import static org.hamcrest.Matchers.*; /** * @@ -63,10 +58,10 @@ public class GeoBoundsIT extends AbstractGeoTestCase { assertThat(geoBounds.getName(), equalTo(aggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft.lat(), equalTo(singleTopLeft.lat())); - assertThat(topLeft.lon(), equalTo(singleTopLeft.lon())); - assertThat(bottomRight.lat(), equalTo(singleBottomRight.lat())); - assertThat(bottomRight.lon(), equalTo(singleBottomRight.lon())); + assertThat(topLeft.lat(), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE)); } public void testSingleValuedField_getProperty() throws Exception { @@ -92,19 +87,19 @@ public class GeoBoundsIT extends AbstractGeoTestCase { assertThat((GeoBounds) global.getProperty(aggName), sameInstance(geobounds)); GeoPoint topLeft = geobounds.topLeft(); GeoPoint bottomRight = geobounds.bottomRight(); - assertThat(topLeft.lat(), equalTo(singleTopLeft.lat())); - assertThat(topLeft.lon(), equalTo(singleTopLeft.lon())); - assertThat(bottomRight.lat(), equalTo(singleBottomRight.lat())); - assertThat(bottomRight.lon(), equalTo(singleBottomRight.lon())); - assertThat((double) global.getProperty(aggName + ".top"), equalTo(singleTopLeft.lat())); - assertThat((double) global.getProperty(aggName + ".left"), equalTo(singleTopLeft.lon())); - assertThat((double) global.getProperty(aggName + ".bottom"), equalTo(singleBottomRight.lat())); - assertThat((double) global.getProperty(aggName + ".right"), equalTo(singleBottomRight.lon())); + assertThat(topLeft.lat(), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE)); + assertThat((double) global.getProperty(aggName + ".top"), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat((double) global.getProperty(aggName + ".left"), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat((double) global.getProperty(aggName + ".bottom"), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat((double) global.getProperty(aggName + ".right"), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE)); } public void testMultiValuedField() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME) - .addAggregation(geoBounds(aggName).field(MULTI_VALUED_FIELD_NAME) + .addAggregation(geoBounds(aggName).field(MULTI_VALUED_FIELD_NAME) .wrapLongitude(false)) .execute().actionGet(); @@ -116,10 +111,10 @@ public class GeoBoundsIT extends AbstractGeoTestCase { assertThat(geoBounds.getName(), equalTo(aggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft.lat(), equalTo(multiTopLeft.lat())); - assertThat(topLeft.lon(), equalTo(multiTopLeft.lon())); - assertThat(bottomRight.lat(), equalTo(multiBottomRight.lat())); - assertThat(bottomRight.lon(), equalTo(multiBottomRight.lon())); + assertThat(topLeft.lat(), closeTo(multiTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(multiTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(multiBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(multiBottomRight.lon(), GEOHASH_TOLERANCE)); } public void testUnmapped() throws Exception { @@ -152,10 +147,10 @@ public class GeoBoundsIT extends AbstractGeoTestCase { assertThat(geoBounds.getName(), equalTo(aggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft.lat(), equalTo(singleTopLeft.lat())); - assertThat(topLeft.lon(), equalTo(singleTopLeft.lon())); - assertThat(bottomRight.lat(), equalTo(singleBottomRight.lat())); - assertThat(bottomRight.lon(), equalTo(singleBottomRight.lon())); + assertThat(topLeft.lat(), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE)); } public void testEmptyAggregation() throws Exception { @@ -191,10 +186,10 @@ public class GeoBoundsIT extends AbstractGeoTestCase { assertThat(geoBounds.getName(), equalTo(aggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft.lat(), equalTo(geoValuesTopLeft.lat())); - assertThat(topLeft.lon(), equalTo(geoValuesTopLeft.lon())); - assertThat(bottomRight.lat(), equalTo(geoValuesBottomRight.lat())); - assertThat(bottomRight.lon(), equalTo(geoValuesBottomRight.lon())); + assertThat(topLeft.lat(), closeTo(geoValuesTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(geoValuesTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(geoValuesBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(geoValuesBottomRight.lon(), GEOHASH_TOLERANCE)); } public void testSingleValuedFieldNearDateLineWrapLongitude() throws Exception { @@ -212,10 +207,10 @@ public class GeoBoundsIT extends AbstractGeoTestCase { assertThat(geoBounds.getName(), equalTo(aggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft.lat(), equalTo(geoValuesTopLeft.lat())); - assertThat(topLeft.lon(), equalTo(geoValuesTopLeft.lon())); - assertThat(bottomRight.lat(), equalTo(geoValuesBottomRight.lat())); - assertThat(bottomRight.lon(), equalTo(geoValuesBottomRight.lon())); + assertThat(topLeft.lat(), closeTo(geoValuesTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(geoValuesTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(geoValuesBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(geoValuesBottomRight.lon(), GEOHASH_TOLERANCE)); } /** @@ -259,9 +254,9 @@ public class GeoBoundsIT extends AbstractGeoTestCase { assertThat(geoBounds.getName(), equalTo(aggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft.lat(), equalTo(1.0)); - assertThat(topLeft.lon(), equalTo(0.0)); - assertThat(bottomRight.lat(), equalTo(1.0)); - assertThat(bottomRight.lon(), equalTo(0.0)); + assertThat(topLeft.lat(), closeTo(1.0, GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(0.0, GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(1.0, GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(0.0, GEOHASH_TOLERANCE)); } -} +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index 9437c288051..b52eec448c5 100644 --- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -20,6 +20,8 @@ package org.elasticsearch.search.builder; import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.Injector; import org.elasticsearch.common.inject.ModulesBuilder; @@ -31,9 +33,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.*; import org.elasticsearch.index.query.AbstractQueryTestCase; import org.elasticsearch.index.query.EmptyQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -292,13 +292,17 @@ public class SearchSourceBuilderTests extends ESTestCase { } public void testFromXContent() throws IOException { - SearchSourceBuilder testBuilder = createSearchSourceBuilder(); - String builderAsString = testBuilder.toString(); - assertParseSearchSource(testBuilder, builderAsString); + SearchSourceBuilder testSearchSourceBuilder = createSearchSourceBuilder(); + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + builder.prettyPrint(); + } + testSearchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertParseSearchSource(testSearchSourceBuilder, builder.bytes()); } - private void assertParseSearchSource(SearchSourceBuilder testBuilder, String builderAsString) throws IOException { - XContentParser parser = XContentFactory.xContent(builderAsString).createParser(builderAsString); + private void assertParseSearchSource(SearchSourceBuilder testBuilder, BytesReference searchSourceAsBytes) throws IOException { + XContentParser parser = XContentFactory.xContent(searchSourceAsBytes).createParser(searchSourceAsBytes); QueryParseContext parseContext = createParseContext(parser); parseContext.reset(parser); if (randomBoolean()) { @@ -418,6 +422,6 @@ public class SearchSourceBuilderTests extends ESTestCase { SearchSourceBuilder builder = new SearchSourceBuilder(); builder.postFilter(EmptyQueryBuilder.PROTOTYPE); String query = "{ \"post_filter\": {} }"; - assertParseSearchSource(builder, query); + assertParseSearchSource(builder, new BytesArray(query)); } } diff --git a/distribution/build.gradle b/distribution/build.gradle index 7770c0d98ab..f2aca91cb86 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -195,3 +195,5 @@ DependencyLicensesTask.configure(project) { mapping from: /lucene-.*/, to: 'lucene' mapping from: /jackson-.*/, to: 'jackson' } + +task run(type:org.elasticsearch.gradle.test.RunTask){} \ No newline at end of file diff --git a/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1711508.jar.sha1 b/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index 7ba588ce118..00000000000 --- a/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1ce5d6dab63f88bd997c2e465ec47efc2891ba5d \ No newline at end of file diff --git a/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1712973.jar.sha1 b/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..2394c85bc48 --- /dev/null +++ b/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +d70e990753cda92e9e817eb7e19baf8a17e663f0 diff --git a/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1711508.jar.sha1 b/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index f741492749d..00000000000 --- a/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8640d74259f60aa4dada4912c9c3ebe772c9818e \ No newline at end of file diff --git a/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1712973.jar.sha1 b/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..6fbfa6e2823 --- /dev/null +++ b/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +68387395739fa891ca728c84b3f79d41d8becc3a diff --git a/distribution/licenses/lucene-core-5.4.0-snapshot-1711508.jar.sha1 b/distribution/licenses/lucene-core-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index f48292917c5..00000000000 --- a/distribution/licenses/lucene-core-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6641f07bcfd6a3d046b83e631e366b706f035c2e \ No newline at end of file diff --git a/distribution/licenses/lucene-core-5.4.0-snapshot-1712973.jar.sha1 b/distribution/licenses/lucene-core-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..bc094272d1c --- /dev/null +++ b/distribution/licenses/lucene-core-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +d984920437acc95a0f488c6df35ad2e846103471 diff --git a/distribution/licenses/lucene-grouping-5.4.0-snapshot-1711508.jar.sha1 b/distribution/licenses/lucene-grouping-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index 856e03e684a..00000000000 --- a/distribution/licenses/lucene-grouping-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b4e39e2b74aea6675b3c4717fa767b84f781d890 \ No newline at end of file diff --git a/distribution/licenses/lucene-grouping-5.4.0-snapshot-1712973.jar.sha1 b/distribution/licenses/lucene-grouping-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..bd5b9fb9d99 --- /dev/null +++ b/distribution/licenses/lucene-grouping-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +eed34253185f1c3cf319d1a19714692471f916b8 diff --git a/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1711508.jar.sha1 b/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index b5626bee99e..00000000000 --- a/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -92aaae41d7f2a7feaa821ce6ad02abc4f8734bc2 \ No newline at end of file diff --git a/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1712973.jar.sha1 b/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..412dbcb7693 --- /dev/null +++ b/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +726b97ce4293dcf3b9e6cd6e11d6c44f21a69e91 diff --git a/distribution/licenses/lucene-join-5.4.0-snapshot-1711508.jar.sha1 b/distribution/licenses/lucene-join-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index 2dee65504ba..00000000000 --- a/distribution/licenses/lucene-join-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e4c5402e04b4875bbe033077de9f6dea16418cb2 \ No newline at end of file diff --git a/distribution/licenses/lucene-join-5.4.0-snapshot-1712973.jar.sha1 b/distribution/licenses/lucene-join-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..3f713ff46c7 --- /dev/null +++ b/distribution/licenses/lucene-join-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +e594cd351b1345f1c0c5c768479942cc5050d04f diff --git a/distribution/licenses/lucene-memory-5.4.0-snapshot-1711508.jar.sha1 b/distribution/licenses/lucene-memory-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index d093ade101a..00000000000 --- a/distribution/licenses/lucene-memory-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9ead1255154fde92384e9f454999ad37e1261e3d \ No newline at end of file diff --git a/distribution/licenses/lucene-memory-5.4.0-snapshot-1712973.jar.sha1 b/distribution/licenses/lucene-memory-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..4e5c2cadb04 --- /dev/null +++ b/distribution/licenses/lucene-memory-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +ed123b7bc78460396899abb90649dad89f7744e0 diff --git a/distribution/licenses/lucene-misc-5.4.0-snapshot-1711508.jar.sha1 b/distribution/licenses/lucene-misc-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index 16d59356d36..00000000000 --- a/distribution/licenses/lucene-misc-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a03e432df9037631a5e9238822927ee4a2167d8d \ No newline at end of file diff --git a/distribution/licenses/lucene-misc-5.4.0-snapshot-1712973.jar.sha1 b/distribution/licenses/lucene-misc-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..7a7cc4769f0 --- /dev/null +++ b/distribution/licenses/lucene-misc-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +ca9cd3d0040328219c0ddfa28cef7fcb6559b98c diff --git a/distribution/licenses/lucene-queries-5.4.0-snapshot-1711508.jar.sha1 b/distribution/licenses/lucene-queries-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index 87503961b39..00000000000 --- a/distribution/licenses/lucene-queries-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -159331ae15424664f70da76f1fb136c1e84ad095 \ No newline at end of file diff --git a/distribution/licenses/lucene-queries-5.4.0-snapshot-1712973.jar.sha1 b/distribution/licenses/lucene-queries-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..b3dd15f6bc7 --- /dev/null +++ b/distribution/licenses/lucene-queries-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +8a814d31b827f7b418b8f3aeb0417c43e2edf533 diff --git a/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1711508.jar.sha1 b/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index 70e78527f21..00000000000 --- a/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9f6e02d1d54db4e7b39f2a9ff337285088ddb0e2 \ No newline at end of file diff --git a/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1712973.jar.sha1 b/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..b944d401c64 --- /dev/null +++ b/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +ecd788599b312a3bbd3aaa9c12d5714e72c1cf1e diff --git a/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1711508.jar.sha1 b/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index 4d551f223bb..00000000000 --- a/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f5e9fb1b996609e18ab7e4aae9ca9438ee66b437 \ No newline at end of file diff --git a/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1712973.jar.sha1 b/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..15f1611fd5a --- /dev/null +++ b/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +50aa4f366fd7832b8a1c2fb50843ee1058805902 diff --git a/distribution/licenses/lucene-spatial-5.4.0-snapshot-1711508.jar.sha1 b/distribution/licenses/lucene-spatial-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index d0b7d27d162..00000000000 --- a/distribution/licenses/lucene-spatial-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c4d8c788084feceac969a2ecea547d42bf0b2715 \ No newline at end of file diff --git a/distribution/licenses/lucene-spatial-5.4.0-snapshot-1712973.jar.sha1 b/distribution/licenses/lucene-spatial-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..9db1d4fb1c3 --- /dev/null +++ b/distribution/licenses/lucene-spatial-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +e301e9e35e6f33e658ccd0ae06b32336b13a935c diff --git a/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1711508.jar.sha1 b/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index 6a98328bb11..00000000000 --- a/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -04c4ead54ca6546ec7447f7f1916077c4767eb3d \ No newline at end of file diff --git a/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1712973.jar.sha1 b/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..e3b5fd6edb6 --- /dev/null +++ b/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +864e2e23f0e64c9d9dd412c0b0aaa36705e37786 diff --git a/distribution/licenses/lucene-suggest-5.4.0-snapshot-1711508.jar.sha1 b/distribution/licenses/lucene-suggest-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index b6113edf88c..00000000000 --- a/distribution/licenses/lucene-suggest-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10d9a389e44de137651d2c87ae401c9d29230bce \ No newline at end of file diff --git a/distribution/licenses/lucene-suggest-5.4.0-snapshot-1712973.jar.sha1 b/distribution/licenses/lucene-suggest-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..f39adec1651 --- /dev/null +++ b/distribution/licenses/lucene-suggest-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +0279e2978e391523da6f5c577d598be0cc94f883 diff --git a/docs/plugins/plugin-script.asciidoc b/docs/plugins/plugin-script.asciidoc index e75155b8675..3f7a30556af 100644 --- a/docs/plugins/plugin-script.asciidoc +++ b/docs/plugins/plugin-script.asciidoc @@ -68,7 +68,7 @@ plugin from GitHub, run one of the following commands: [source,shell] ----------------------------------- sudo bin/plugin install lmenezes/elasticsearch-kopf <1> -sudo bin/plugin install lmenezes/elasticsearch-kopf/1.x <2> +sudo bin/plugin install lmenezes/elasticsearch-kopf/2.x <2> ----------------------------------- <1> Installs the latest version from GitHub. <2> Installs the 1.x version from GitHub. @@ -104,6 +104,15 @@ For instance, to install a plugin from your local file system, you could run: sudo bin/plugin install file:///path/to/plugin.zip ----------------------------------- +The plugin script will refuse to talk to an HTTPS URL with an untrusted +certificate. To use a self-signed HTTPS cert, you will need to add the CA cert +to a local Java truststore and pass the location to the script as follows: + +[source,shell] +----------------------------------- +sudo bin/plugin -Djavax.net.ssl.trustStore=/path/to/trustStore.jks install https://.... +----------------------------------- + [[listing-removing]] === Listing and Removing Installed Plugins diff --git a/docs/reference/index-modules/allocation/total_shards.asciidoc b/docs/reference/index-modules/allocation/total_shards.asciidoc index 3e1b3ab16e8..691ab8d937d 100644 --- a/docs/reference/index-modules/allocation/total_shards.asciidoc +++ b/docs/reference/index-modules/allocation/total_shards.asciidoc @@ -14,10 +14,17 @@ number of shards from a single index allowed per node: The maximum number of shards (replicas and primaries) that will be allocated to a single node. Defaults to unbounded. +You can also limit the amount of shards a node can have regardless of the index: + +`cluster.routing.allocation.total_shards_per_node`:: + + The maximum number of shards (replicas and primaries) that will be + allocated to a single node globally. Defaults to unbounded (-1). + [WARNING] ======================================= -This setting imposes a hard limit which can result in some shards not -being allocated. +Thess setting impose a hard limit which can result in some shards not being +allocated. Use with caution. ======================================= diff --git a/docs/reference/mapping/fields/all-field.asciidoc b/docs/reference/mapping/fields/all-field.asciidoc index 00c8d3b245b..e206dcd125f 100644 --- a/docs/reference/mapping/fields/all-field.asciidoc +++ b/docs/reference/mapping/fields/all-field.asciidoc @@ -2,7 +2,7 @@ === `_all` field The `_all` field is a special _catch-all_ field which concatenates the values -of all of the other fields into one big string, which is then +of all of the other fields into one big string, using space as a delimiter, which is then <> and indexed, but not stored. This means that it can be searched, but not retrieved. diff --git a/docs/reference/mapping/params/doc-values.asciidoc b/docs/reference/mapping/params/doc-values.asciidoc index d47f7cbfe9f..ea8dfd08bb7 100644 --- a/docs/reference/mapping/params/doc-values.asciidoc +++ b/docs/reference/mapping/params/doc-values.asciidoc @@ -12,8 +12,10 @@ documents, we need to be able to look up the document and find the terms that is has in a field. Doc values are the on-disk data structure, built at document index time, which -makes this data access pattern possible. Doc values are supported on almost -all field types, with the __notable exception of `analyzed` string fields__. +makes this data access pattern possible. They store the same values as the +`_source` but in a column-oriented fashion that is way more efficient for +sorting and aggregations. Doc values are supported on almost all field types, +with the __notable exception of `analyzed` string fields__. All fields which support doc values have them enabled by default. If you are sure that you don't need to sort or aggregate on a field, or access the field diff --git a/docs/reference/mapping/types/binary.asciidoc b/docs/reference/mapping/types/binary.asciidoc index ff76fbebf90..4e5f6b4bc27 100644 --- a/docs/reference/mapping/types/binary.asciidoc +++ b/docs/reference/mapping/types/binary.asciidoc @@ -40,8 +40,9 @@ The following parameters are accepted by `binary` fields: <>:: - Can the field value be used for sorting, aggregations, or scripting? - Accepts `true` or `false` (default). + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. <>:: diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc index 5ebcc651d09..9ff1aa13dde 100644 --- a/docs/reference/mapping/types/boolean.asciidoc +++ b/docs/reference/mapping/types/boolean.asciidoc @@ -98,8 +98,9 @@ The following parameters are accepted by `boolean` fields: <>:: - Can the field value be used for sorting, aggregations, or scripting? - Accepts `true` (default) or `false`. + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. <>:: diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc index c8067a89fdf..118c1a85d4f 100644 --- a/docs/reference/mapping/types/date.asciidoc +++ b/docs/reference/mapping/types/date.asciidoc @@ -97,8 +97,9 @@ The following parameters are accepted by `date` fields: <>:: - Can the field value be used for sorting, aggregations, or scripting? - Accepts `true` (default) or `false`. + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. <>:: diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc index 0049d8f93ac..ad7230e68d6 100644 --- a/docs/reference/mapping/types/geo-point.asciidoc +++ b/docs/reference/mapping/types/geo-point.asciidoc @@ -108,8 +108,9 @@ The following parameters are accepted by `geo_point` fields: <>:: - Can the field value be used for sorting, aggregations, or scripting? - Accepts `true` (default) or `false`. + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. <>:: diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc index 9610466acc2..9b7443ef60a 100644 --- a/docs/reference/mapping/types/ip.asciidoc +++ b/docs/reference/mapping/types/ip.asciidoc @@ -54,8 +54,9 @@ The following parameters are accepted by `ip` fields: <>:: - Can the field value be used for sorting, aggregations, or scripting? - Accepts `true` (default) or `false`. + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. <>:: diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index f04efa16583..77f5808e6b0 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -52,8 +52,9 @@ The following parameters are accepted by numeric types: <>:: - Can the field value be used for sorting, aggregations, or scripting? - Accepts `true` (default) or `false`. + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. <>:: diff --git a/docs/reference/mapping/types/string.asciidoc b/docs/reference/mapping/types/string.asciidoc index d5d7b7a0fce..95c682c696f 100644 --- a/docs/reference/mapping/types/string.asciidoc +++ b/docs/reference/mapping/types/string.asciidoc @@ -82,9 +82,10 @@ The following parameters are accepted by `string` fields: <>:: - Can the field use on-disk index-time doc values for sorting, aggregations, - or scripting? Accepts `true` or `false`. Defaults to `true` for - `not_analyzed` fields. Analyzed fields do not support doc values. + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + or `false`. Defaults to `true` for `not_analyzed` fields. Analyzed fields + do not support doc values. <>:: diff --git a/docs/reference/mapping/types/token-count.asciidoc b/docs/reference/mapping/types/token-count.asciidoc index 6c1b93c34d9..03d2308a4ca 100644 --- a/docs/reference/mapping/types/token-count.asciidoc +++ b/docs/reference/mapping/types/token-count.asciidoc @@ -75,8 +75,9 @@ The following parameters are accepted by `token_count` fields: <>:: - Can the field value be used for sorting, aggregations, or scripting? - Accepts `true` (default) or `false`. + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. <>:: diff --git a/docs/reference/modules.asciidoc b/docs/reference/modules.asciidoc index 0beb88a4abc..09ffb06fb68 100644 --- a/docs/reference/modules.asciidoc +++ b/docs/reference/modules.asciidoc @@ -70,8 +70,8 @@ The modules in this section are: <>:: - A tribe node joins one ore more clusters and act as a federated - client accross them. + A tribe node joins one or more clusters and acts as a federated + client across them. -- diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc index 89033aa1f43..325ae0d94a6 100644 --- a/docs/reference/search/request-body.asciidoc +++ b/docs/reference/search/request-body.asciidoc @@ -94,6 +94,46 @@ parameter named `source`. Both HTTP GET and HTTP POST can be used to execute search with body. Since not all clients support GET with body, POST is allowed as well. +[float] +=== Fast check for any matching docs + +In case we only want to know if there are any documents matching a +specific query, we can set the `size` to `0` to indicate that we are not +interested in the search results. Also we can set `terminate_after` to `1` +to indicate that the query execution can be terminated whenever the first +matching document was found (per shard). + +[source,js] +-------------------------------------------------- +$ curl -XGET 'http://localhost:9200/_search?q=tag:wow&size=0&terminate_after=1' +-------------------------------------------------- + +The response will not contain any hits as the `size` was set to `0`. The +`hits.total` will be either equal to `0`, indicating that there were no +matching documents, or greater than `0` meaning that there were at least +as many documents matching the query when it was early terminated. +Also if the query was terminated early, the `terminated_early` flag will +be set to `true` in the response. + +[source,js] +-------------------------------------------------- +{ + "took": 3, + "timed_out": false, + "terminated_early": true, + "_shards": { + "total": 1, + "successful": 1, + "failed": 0 + }, + "hits": { + "total": 1, + "max_score": 0, + "hits": [] + } +} +-------------------------------------------------- + include::request/query.asciidoc[] diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc index 4a5a867f08c..cad6f5a63b8 100644 --- a/docs/reference/search/search.asciidoc +++ b/docs/reference/search/search.asciidoc @@ -49,4 +49,3 @@ Or even search across all indices and all types: -------------------------------------------------- $ curl -XGET 'http://localhost:9200/_search?q=tag:wow' -------------------------------------------------- - diff --git a/gradle.properties b/gradle.properties deleted file mode 100644 index 95510ac3653..00000000000 --- a/gradle.properties +++ /dev/null @@ -1,3 +0,0 @@ -group=org.elasticsearch -version=3.0.0-SNAPSHOT -luceneVersion=5.4.0-snapshot-1711508 diff --git a/plugins/analysis-icu/build.gradle b/plugins/analysis-icu/build.gradle index 8cf0cea118d..a662f727eeb 100644 --- a/plugins/analysis-icu/build.gradle +++ b/plugins/analysis-icu/build.gradle @@ -24,6 +24,7 @@ esplugin { dependencies { compile "org.apache.lucene:lucene-analyzers-icu:${versions.lucene}" + compile 'com.ibm.icu:icu4j:54.1' } dependencyLicenses { diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1711508.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index 8c09ac37be1..00000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8843675db98498fd858c071644238420cb8d270a \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1712973.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..71a601bbaf3 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +8cc60f870ebfb5e27f01810032c6d57765978b37 diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1711508.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index ad36332c96c..00000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b2a0b8143e660826b8605cf58de818a6ce423f47 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1712973.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..88f2b91e2dd --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +de798609f67720bee73c0a5139b3b87cb6d0aa78 diff --git a/plugins/analysis-phonetic/build.gradle b/plugins/analysis-phonetic/build.gradle index c3a0f12007a..13898be05a9 100644 --- a/plugins/analysis-phonetic/build.gradle +++ b/plugins/analysis-phonetic/build.gradle @@ -24,6 +24,7 @@ esplugin { dependencies { compile "org.apache.lucene:lucene-analyzers-phonetic:${versions.lucene}" + compile "commons-codec:commons-codec:${versions.commonscodec}" } dependencyLicenses { diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1711508.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index 308e1edff4e..00000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -414bc537c247f07eb1b34ff943fc0ce4c80aab8c \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1712973.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..ac3eaa2f1b4 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +bfb5d0a5626532e2256f5a430a2b9fca7243c9b9 diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1711508.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index 96ca2605575..00000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f2c2e3907a66825e042eb8f664f210d949f05108 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1712973.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..dd4b11d2f9d --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +286fa90cd48c8500f99852fc597a08bc00cf3302 diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1711508.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index 98240f9b732..00000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7059da88bf88df123c62d917aeed4c144d62665f \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1712973.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..6db14acb660 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +37598eb36c271b93707d9f106f84a4dcc60b97c9 diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java index 83a30156ee7..df57aca1668 100644 --- a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java +++ b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java @@ -27,13 +27,7 @@ import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; -import org.elasticsearch.action.search.ClearScrollResponse; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchScrollRequest; -import org.elasticsearch.action.search.ShardSearchFailure; -import org.elasticsearch.action.search.TransportSearchAction; -import org.elasticsearch.action.search.TransportSearchScrollAction; +import org.elasticsearch.action.search.*; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.Client; @@ -48,10 +42,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -109,8 +100,11 @@ public class TransportDeleteByQueryAction extends HandledTransportAction() { + scrollAction.execute(new SearchScrollRequest(request).scrollId(scrollId).scroll(request.scroll()), new ActionListener() { @Override public void onResponse(SearchResponse scrollResponse) { deleteHits(scrollId, scrollResponse); @@ -197,9 +192,9 @@ public class TransportDeleteByQueryAction extends HandledTransportAction() { + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(request); + clearScrollRequest.addScrollId(scrollId); + client.clearScroll(clearScrollRequest, new ActionListener() { @Override public void onResponse(ClearScrollResponse clearScrollResponse) { logger.trace("scroll id [{}] cleared", scrollId); diff --git a/plugins/discovery-azure/build.gradle b/plugins/discovery-azure/build.gradle index d16fbf756ed..d72c203d089 100644 --- a/plugins/discovery-azure/build.gradle +++ b/plugins/discovery-azure/build.gradle @@ -22,14 +22,33 @@ esplugin { classname 'org.elasticsearch.plugin.discovery.azure.AzureDiscoveryPlugin' } +versions << [ + 'azure': '0.7.0', + 'jersey': '1.13' +] + dependencies { - compile('com.microsoft.azure:azure-management-compute:0.7.0') { - exclude group: 'stax', module: 'stax-api' - } - compile('com.microsoft.azure:azure-management:0.7.0') { - exclude group: 'stax', module: 'stax-api' - } + compile "com.microsoft.azure:azure-management-compute:${versions.azure}" + compile "com.microsoft.azure:azure-management:${versions.azure}" + compile "com.microsoft.azure:azure-core:${versions.azure}" compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" + compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" + compile "commons-logging:commons-logging:${versions.commonslogging}" + compile "commons-codec:commons-codec:${versions.commonscodec}" + compile 'javax.mail:mail:1.4.5' + compile 'javax.activation:activation:1.1' + compile 'javax.inject:javax.inject:1' + compile "com.sun.jersey:jersey-client:${versions.jersey}" + compile "com.sun.jersey:jersey-core:${versions.jersey}" + compile "com.sun.jersey:jersey-json:${versions.jersey}" + compile 'org.codehaus.jettison:jettison:1.1' + compile 'com.sun.xml.bind:jaxb-impl:2.2.3-1' + compile 'javax.xml.bind:jaxb-api:2.2.2' + compile 'javax.xml.stream:stax-api:1.0-2' + compile 'org.codehaus.jackson:jackson-core-asl:1.9.2' + compile 'org.codehaus.jackson:jackson-mapper-asl:1.9.2' + compile 'org.codehaus.jackson:jackson-jaxrs:1.9.2' + compile 'org.codehaus.jackson:jackson-xc:1.9.2' } dependencyLicenses { diff --git a/plugins/discovery-azure/licenses/commons-codec-1.10.jar.sha1 b/plugins/discovery-azure/licenses/commons-codec-1.10.jar.sha1 new file mode 100644 index 00000000000..3fe8682a1b0 --- /dev/null +++ b/plugins/discovery-azure/licenses/commons-codec-1.10.jar.sha1 @@ -0,0 +1 @@ +4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/discovery-azure/licenses/commons-codec-1.6.jar.sha1 b/plugins/discovery-azure/licenses/commons-codec-1.6.jar.sha1 deleted file mode 100644 index bf78aff7364..00000000000 --- a/plugins/discovery-azure/licenses/commons-codec-1.6.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b7f0fc8f61ecadeb3695f0b9464755eee44374d4 diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index 2b0365aff59..25706619c15 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -22,9 +22,19 @@ esplugin { classname 'org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin' } +versions << [ + 'aws': '1.10.33' +] + dependencies { - compile 'com.amazonaws:aws-java-sdk-ec2:1.10.19' + compile "com.amazonaws:aws-java-sdk-ec2:${versions.aws}" + compile "com.amazonaws:aws-java-sdk-core:${versions.aws}" compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" + compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" + compile "commons-logging:commons-logging:${versions.commonslogging}" + compile "commons-codec:commons-codec:${versions.commonscodec}" + compile 'com.fasterxml.jackson.core:jackson-databind:2.5.3' + compile 'com.fasterxml.jackson.core:jackson-annotations:2.5.0' } dependencyLicenses { diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.19.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.19.jar.sha1 deleted file mode 100644 index 66e418e6fb2..00000000000 --- a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.19.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b53f650323b7242dcced25b679f3e9aa4b494da5 diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.33.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.33.jar.sha1 new file mode 100644 index 00000000000..332a8f01035 --- /dev/null +++ b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.33.jar.sha1 @@ -0,0 +1 @@ +fabedbbe2b834b1add150b6a38395c5ef7380168 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.19.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.19.jar.sha1 deleted file mode 100644 index 26fa78d2fd4..00000000000 --- a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.19.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -50ba7eb31719be1260bdae51cf69340df2d91ec4 diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.33.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.33.jar.sha1 new file mode 100644 index 00000000000..4737b80b3f2 --- /dev/null +++ b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.33.jar.sha1 @@ -0,0 +1 @@ +202f6b5dbc196e355d50c131b0fd34969bfd89e6 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/commons-codec-1.10.jar.sha1 b/plugins/discovery-ec2/licenses/commons-codec-1.10.jar.sha1 new file mode 100644 index 00000000000..3fe8682a1b0 --- /dev/null +++ b/plugins/discovery-ec2/licenses/commons-codec-1.10.jar.sha1 @@ -0,0 +1 @@ +4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/commons-codec-1.6.jar.sha1 b/plugins/discovery-ec2/licenses/commons-codec-1.6.jar.sha1 deleted file mode 100644 index bf78aff7364..00000000000 --- a/plugins/discovery-ec2/licenses/commons-codec-1.6.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b7f0fc8f61ecadeb3695f0b9464755eee44374d4 diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java index 6b73a71e0be..a95d1a73a75 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java @@ -19,7 +19,6 @@ package org.elasticsearch.plugin.discovery.ec2; -import org.elasticsearch.SpecialPermission; import org.elasticsearch.cloud.aws.AwsEc2ServiceImpl; import org.elasticsearch.cloud.aws.Ec2Module; import org.elasticsearch.common.component.LifecycleComponent; @@ -32,8 +31,6 @@ import org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider; import org.elasticsearch.discovery.ec2.Ec2Discovery; import org.elasticsearch.plugins.Plugin; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Collection; @@ -41,27 +38,6 @@ import java.util.Collection; * */ public class Ec2DiscoveryPlugin extends Plugin { - - static { - // This internal config is deserialized but with wrong access modifiers, - // cannot work without suppressAccessChecks permission right now. We force - // a one time load with elevated privileges as a workaround. - SecurityManager sm = System.getSecurityManager(); - if (sm != null) { - sm.checkPermission(new SpecialPermission()); - } - AccessController.doPrivileged(new PrivilegedAction() { - @Override - public Void run() { - try { - Class.forName("com.amazonaws.internal.config.InternalConfig$Factory"); - } catch (ClassNotFoundException e) { - throw new RuntimeException("Unable to initialize internal aws config", e); - } - return null; - } - }); - } private final Settings settings; protected final ESLogger logger = Loggers.getLogger(Ec2DiscoveryPlugin.class); diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java index 7213a3ace6d..47e2554dcd4 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java @@ -1355,4 +1355,9 @@ public class AmazonEC2Mock implements AmazonEC2 { public ResponseMetadata getCachedResponseMetadata(AmazonWebServiceRequest request) { throw new UnsupportedOperationException("Not supported in mock"); } + + @Override + public ModifySpotFleetRequestResult modifySpotFleetRequest(ModifySpotFleetRequestRequest modifySpotFleetRequestRequest) throws AmazonServiceException, AmazonClientException { + throw new UnsupportedOperationException("Not supported in mock"); + } } diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle index da9cdb94aae..4e6ade8788f 100644 --- a/plugins/discovery-gce/build.gradle +++ b/plugins/discovery-gce/build.gradle @@ -4,11 +4,21 @@ esplugin { classname 'org.elasticsearch.plugin.discovery.gce.GceDiscoveryPlugin' } +versions << [ + 'google': '1.20.0' +] + dependencies { - compile('com.google.apis:google-api-services-compute:v1-rev71-1.20.0') { - exclude group: 'com.google.guava', module: 'guava-jdk5' - } + compile "com.google.apis:google-api-services-compute:v1-rev71-${versions.google}" + compile "com.google.api-client:google-api-client:${versions.google}" + compile "com.google.oauth-client:google-oauth-client:${versions.google}" + compile "com.google.http-client:google-http-client:${versions.google}" + compile "com.google.http-client:google-http-client-jackson2:${versions.google}" + compile 'com.google.code.findbugs:jsr305:1.3.9' compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" + compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" + compile "commons-logging:commons-logging:${versions.commonslogging}" + compile "commons-codec:commons-codec:${versions.commonscodec}" } dependencyLicenses { diff --git a/plugins/discovery-gce/licenses/commons-codec-1.10.jar.sha1 b/plugins/discovery-gce/licenses/commons-codec-1.10.jar.sha1 new file mode 100644 index 00000000000..3fe8682a1b0 --- /dev/null +++ b/plugins/discovery-gce/licenses/commons-codec-1.10.jar.sha1 @@ -0,0 +1 @@ +4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/commons-codec-1.6.jar.sha1 b/plugins/discovery-gce/licenses/commons-codec-1.6.jar.sha1 deleted file mode 100644 index bf78aff7364..00000000000 --- a/plugins/discovery-gce/licenses/commons-codec-1.6.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b7f0fc8f61ecadeb3695f0b9464755eee44374d4 diff --git a/plugins/lang-expression/build.gradle b/plugins/lang-expression/build.gradle index 12a99bb705f..9f62e34687d 100644 --- a/plugins/lang-expression/build.gradle +++ b/plugins/lang-expression/build.gradle @@ -24,6 +24,9 @@ esplugin { dependencies { compile "org.apache.lucene:lucene-expressions:${versions.lucene}" + compile 'org.antlr:antlr4-runtime:4.5.1-1' + compile 'org.ow2.asm:asm:5.0.4' + compile 'org.ow2.asm:asm-commons:5.0.4' } dependencyLicenses { diff --git a/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1711508.jar.sha1 b/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1711508.jar.sha1 deleted file mode 100644 index 7ac4891745d..00000000000 --- a/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1711508.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -627b73ddbbbd03746123751b36a674d81308a046 \ No newline at end of file diff --git a/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1712973.jar.sha1 b/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1712973.jar.sha1 new file mode 100644 index 00000000000..b52643bf786 --- /dev/null +++ b/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1712973.jar.sha1 @@ -0,0 +1 @@ +71f1017b9dff2336be8389ad09c506cfad15b5bf diff --git a/plugins/mapper-attachments/README.md b/plugins/mapper-attachments/README.md new file mode 100644 index 00000000000..7114ad7b2cf --- /dev/null +++ b/plugins/mapper-attachments/README.md @@ -0,0 +1,396 @@ +Mapper Attachments Type for Elasticsearch +========================================= + +The mapper attachments plugin lets Elasticsearch index file attachments in common formats (such as PPT, XLS, PDF) using the Apache text extraction library [Tika](http://lucene.apache.org/tika/). + +In practice, the plugin adds the `attachment` type when mapping properties so that documents can be populated with file attachment contents (encoded as `base64`). + +Installation +------------ + +In order to install the plugin, run: + +```sh +bin/plugin install mapper-attachments +``` + +Hello, world +------------ + +Create a property mapping using the new type `attachment`: + +```javascript +POST /trying-out-mapper-attachments +{ + "mappings": { + "person": { + "properties": { + "cv": { "type": "attachment" } +}}}} +``` + +Index a new document populated with a `base64`-encoded attachment: + +```javascript +POST /trying-out-mapper-attachments/person/1 +{ + "cv": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=" +} +``` + +Search for the document using words in the attachment: + +```javascript +POST /trying-out-mapper-attachments/person/_search +{ + "query": { + "query_string": { + "query": "ipsum" +}}} +``` + +If you get a hit for your indexed document, the plugin should be installed and working. + +Usage +------------------------ + +Using the attachment type is simple, in your mapping JSON, simply set a certain JSON element as attachment, for example: + +```javascript +PUT /test +PUT /test/person/_mapping +{ + "person" : { + "properties" : { + "my_attachment" : { "type" : "attachment" } + } + } +} +``` + +In this case, the JSON to index can be: + +```javascript +PUT /test/person/1 +{ + "my_attachment" : "... base64 encoded attachment ..." +} +``` + +Or it is possible to use more elaborated JSON if content type, resource name or language need to be set explicitly: + +``` +PUT /test/person/1 +{ + "my_attachment" : { + "_content_type" : "application/pdf", + "_name" : "resource/name/of/my.pdf", + "_language" : "en", + "_content" : "... base64 encoded attachment ..." + } +} +``` + +The `attachment` type not only indexes the content of the doc in `content` sub field, but also automatically adds meta +data on the attachment as well (when available). + +The metadata supported are: + +* `date` +* `title` +* `name` only available if you set `_name` see above +* `author` +* `keywords` +* `content_type` +* `content_length` is the original content_length before text extraction (aka file size) +* `language` + +They can be queried using the "dot notation", for example: `my_attachment.author`. + +Both the meta data and the actual content are simple core type mappers (string, date, ...), thus, they can be controlled +in the mappings. For example: + +```javascript +PUT /test/person/_mapping +{ + "person" : { + "properties" : { + "file" : { + "type" : "attachment", + "fields" : { + "content" : {"index" : "no"}, + "title" : {"store" : "yes"}, + "date" : {"store" : "yes"}, + "author" : {"analyzer" : "myAnalyzer"}, + "keywords" : {"store" : "yes"}, + "content_type" : {"store" : "yes"}, + "content_length" : {"store" : "yes"}, + "language" : {"store" : "yes"} + } + } + } + } +} +``` + +In the above example, the actual content indexed is mapped under `fields` name `content`, and we decide not to index it, so +it will only be available in the `_all` field. The other fields map to their respective metadata names, but there is no +need to specify the `type` (like `string` or `date`) since it is already known. + +Copy To feature +--------------- + +If you want to use [copy_to](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#copy-to) +feature, you need to define it on each sub-field you want to copy to another field: + +```javascript +PUT /test/person/_mapping +{ + "person": { + "properties": { + "file": { + "type": "attachment", + "fields": { + "content": { + "type": "string", + "copy_to": "copy" + } + } + }, + "copy": { + "type": "string" + } + } + } +} +``` + +In this example, the extracted content will be copy as well to `copy` field. + +Querying or accessing metadata +------------------------------ + +If you need to query on metadata fields, use the attachment field name dot the metadata field. For example: + +``` +DELETE /test +PUT /test +PUT /test/person/_mapping +{ + "person": { + "properties": { + "file": { + "type": "attachment", + "fields": { + "content_type": { + "type": "string", + "store": true + } + } + } + } + } +} +PUT /test/person/1?refresh=true +{ + "file": "IkdvZCBTYXZlIHRoZSBRdWVlbiIgKGFsdGVybmF0aXZlbHkgIkdvZCBTYXZlIHRoZSBLaW5nIg==" +} +GET /test/person/_search +{ + "fields": [ "file.content_type" ], + "query": { + "match": { + "file.content_type": "text plain" + } + } +} +``` + +Will give you: + +``` +{ + "took": 2, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "failed": 0 + }, + "hits": { + "total": 1, + "max_score": 0.16273327, + "hits": [ + { + "_index": "test", + "_type": "person", + "_id": "1", + "_score": 0.16273327, + "fields": { + "file.content_type": [ + "text/plain; charset=ISO-8859-1" + ] + } + } + ] + } +} +``` + +Indexed Characters +------------------ + +By default, `100000` characters are extracted when indexing the content. This default value can be changed by setting +the `index.mapping.attachment.indexed_chars` setting. It can also be provided on a per document indexed using the +`_indexed_chars` parameter. `-1` can be set to extract all text, but note that all the text needs to be allowed to be +represented in memory: + +``` +PUT /test/person/1 +{ + "my_attachment" : { + "_indexed_chars" : -1, + "_content" : "... base64 encoded attachment ..." + } +} +``` + +Metadata parsing error handling +------------------------------- + +While extracting metadata content, errors could happen for example when parsing dates. +Parsing errors are ignored so your document is indexed. + +You can disable this feature by setting the `index.mapping.attachment.ignore_errors` setting to `false`. + +Language Detection +------------------ + +By default, language detection is disabled (`false`) as it could come with a cost. +This default value can be changed by setting the `index.mapping.attachment.detect_language` setting. +It can also be provided on a per document indexed using the `_detect_language` parameter. + +Note that you can force language using `_language` field when sending your actual document: + +```javascript +{ + "my_attachment" : { + "_language" : "en", + "_content" : "... base64 encoded attachment ..." + } +} +``` + +Highlighting attachments +------------------------ + +If you want to highlight your attachment content, you will need to set `"store": true` and `"term_vector":"with_positions_offsets"` +for your attachment field. Here is a full script which does it: + +``` +DELETE /test +PUT /test +PUT /test/person/_mapping +{ + "person": { + "properties": { + "file": { + "type": "attachment", + "fields": { + "content": { + "type": "string", + "term_vector":"with_positions_offsets", + "store": true + } + } + } + } + } +} +PUT /test/person/1?refresh=true +{ + "file": "IkdvZCBTYXZlIHRoZSBRdWVlbiIgKGFsdGVybmF0aXZlbHkgIkdvZCBTYXZlIHRoZSBLaW5nIg==" +} +GET /test/person/_search +{ + "fields": [], + "query": { + "match": { + "file.content": "king queen" + } + }, + "highlight": { + "fields": { + "file.content": { + } + } + } +} +``` + +It gives back: + +```js +{ + "took": 9, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "failed": 0 + }, + "hits": { + "total": 1, + "max_score": 0.13561106, + "hits": [ + { + "_index": "test", + "_type": "person", + "_id": "1", + "_score": 0.13561106, + "highlight": { + "file.content": [ + "\"God Save the Queen\" (alternatively \"God Save the King\"\n" + ] + } + } + ] + } +} +``` + +Stand alone runner +------------------ + +If you want to run some tests within your IDE, you can use `StandaloneRunner` class. +It accepts arguments: + +* `-u file://URL/TO/YOUR/DOC` +* `--size` set extracted size (default to mapper attachment size) +* `BASE64` encoded binary + +Example: + +```sh +StandaloneRunner BASE64Text +StandaloneRunner -u /tmp/mydoc.pdf +StandaloneRunner -u /tmp/mydoc.pdf --size 1000000 +``` + +It produces something like: + +``` +## Extracted text +--------------------- BEGIN ----------------------- +This is the extracted text +---------------------- END ------------------------ +## Metadata +- author: null +- content_length: null +- content_type: application/pdf +- date: null +- keywords: null +- language: null +- name: null +- title: null +``` diff --git a/plugins/mapper-attachments/build.gradle b/plugins/mapper-attachments/build.gradle new file mode 100644 index 00000000000..e14cf543043 --- /dev/null +++ b/plugins/mapper-attachments/build.gradle @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The mapper attachments plugin adds the attachment type to Elasticsearch using Apache Tika.' + classname 'org.elasticsearch.mapper.attachments.MapperAttachmentsPlugin' +} + +versions << [ + 'tika': '1.11', + 'pdfbox': '1.8.10', + 'bouncycastle': '1.52', + 'poi': '3.13' +] + +dependencies { + // mandatory for tika + compile "org.apache.tika:tika-core:${versions.tika}" + compile "org.apache.tika:tika-parsers:${versions.tika}" + compile 'commons-io:commons-io:2.4' + + // character set detection + compile 'com.googlecode.juniversalchardet:juniversalchardet:1.0.3' + + // external parser libraries + // HTML + compile 'org.ccil.cowan.tagsoup:tagsoup:1.2.1' + // Adobe PDF + compile "org.apache.pdfbox:pdfbox:${versions.pdfbox}" + compile "org.apache.pdfbox:fontbox:${versions.pdfbox}" + compile "org.apache.pdfbox:jempbox:${versions.pdfbox}" + compile "commons-logging:commons-logging:${versions.commonslogging}" + compile "org.bouncycastle:bcmail-jdk15on:${versions.bouncycastle}" + compile "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}" + compile "org.bouncycastle:bcpkix-jdk15on:${versions.bouncycastle}" + // OpenOffice + compile "org.apache.poi:poi-ooxml:${versions.poi}" + compile "org.apache.poi:poi:${versions.poi}" + compile "org.apache.poi:poi-ooxml-schemas:${versions.poi}" + compile "commons-codec:commons-codec:${versions.commonscodec}" + compile 'org.apache.xmlbeans:xmlbeans:2.6.0' + compile 'stax:stax-api:1.0.1' + // MS Office + compile "org.apache.poi:poi-scratchpad:${versions.poi}" + // Apple iWork + compile 'org.apache.commons:commons-compress:1.10' +} + +compileJava.options.compilerArgs << '-Xlint:-cast,-deprecation,-rawtypes' + +forbiddenPatterns { + exclude '**/*.docx' + exclude '**/*.pdf' + exclude '**/*.epub' +} diff --git a/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.52.jar.sha1 b/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.52.jar.sha1 new file mode 100644 index 00000000000..de084c948f4 --- /dev/null +++ b/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.52.jar.sha1 @@ -0,0 +1 @@ +4995a870400e1554d1c7ed2afcb5d198fae12db9 diff --git a/plugins/mapper-attachments/licenses/bcmail-jdk15on-LICENSE.txt b/plugins/mapper-attachments/licenses/bcmail-jdk15on-LICENSE.txt new file mode 100644 index 00000000000..dbba1dd7829 --- /dev/null +++ b/plugins/mapper-attachments/licenses/bcmail-jdk15on-LICENSE.txt @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2000 - 2013 The Legion of the Bouncy Castle Inc. + (http://www.bouncycastle.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/plugins/mapper-attachments/licenses/bcmail-jdk15on-NOTICE.txt b/plugins/mapper-attachments/licenses/bcmail-jdk15on-NOTICE.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.52.jar.sha1 b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.52.jar.sha1 new file mode 100644 index 00000000000..489ceeaaf36 --- /dev/null +++ b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.52.jar.sha1 @@ -0,0 +1 @@ +b8ffac2bbc6626f86909589c8cc63637cc936504 diff --git a/plugins/mapper-attachments/licenses/bcpkix-jdk15on-LICENSE.txt b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-LICENSE.txt new file mode 100644 index 00000000000..e1fc4a1506d --- /dev/null +++ b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-LICENSE.txt @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2000 - 2013 The Legion of the Bouncy Castle Inc. + (http://www.bouncycastle.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/plugins/mapper-attachments/licenses/bcpkix-jdk15on-NOTICE.txt b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-NOTICE.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.52.jar.sha1 b/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.52.jar.sha1 new file mode 100644 index 00000000000..14ecc1be40b --- /dev/null +++ b/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.52.jar.sha1 @@ -0,0 +1 @@ +88a941faf9819d371e3174b5ed56a3f3f7d73269 diff --git a/plugins/mapper-attachments/licenses/bcprov-jdk15on-LICENSE.txt b/plugins/mapper-attachments/licenses/bcprov-jdk15on-LICENSE.txt new file mode 100644 index 00000000000..e1fc4a1506d --- /dev/null +++ b/plugins/mapper-attachments/licenses/bcprov-jdk15on-LICENSE.txt @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2000 - 2013 The Legion of the Bouncy Castle Inc. + (http://www.bouncycastle.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/plugins/mapper-attachments/licenses/bcprov-jdk15on-NOTICE.txt b/plugins/mapper-attachments/licenses/bcprov-jdk15on-NOTICE.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/licenses/commons-codec-1.10.jar.sha1 b/plugins/mapper-attachments/licenses/commons-codec-1.10.jar.sha1 new file mode 100644 index 00000000000..3fe8682a1b0 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-codec-1.10.jar.sha1 @@ -0,0 +1 @@ +4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/commons-codec-LICENSE.txt b/plugins/mapper-attachments/licenses/commons-codec-LICENSE.txt new file mode 100644 index 00000000000..57bc88a15a0 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-codec-LICENSE.txt @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/plugins/mapper-attachments/licenses/commons-codec-NOTICE.txt b/plugins/mapper-attachments/licenses/commons-codec-NOTICE.txt new file mode 100644 index 00000000000..72eb32a9024 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-codec-NOTICE.txt @@ -0,0 +1,5 @@ +Apache Commons CLI +Copyright 2001-2009 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/mapper-attachments/licenses/commons-compress-1.10.jar.sha1 b/plugins/mapper-attachments/licenses/commons-compress-1.10.jar.sha1 new file mode 100644 index 00000000000..65c74b9a88f --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-compress-1.10.jar.sha1 @@ -0,0 +1 @@ +5eeb27c57eece1faf2d837868aeccc94d84dcc9a \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/commons-compress-LICENSE.txt b/plugins/mapper-attachments/licenses/commons-compress-LICENSE.txt new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-compress-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/mapper-attachments/licenses/commons-compress-NOTICE.txt b/plugins/mapper-attachments/licenses/commons-compress-NOTICE.txt new file mode 100644 index 00000000000..edd2f2c78ee --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-compress-NOTICE.txt @@ -0,0 +1,11 @@ +Apache Commons Compress +Copyright 2002-2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +The files in the package org.apache.commons.compress.archivers.sevenz +were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), +which has been placed in the public domain: + +"LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) diff --git a/plugins/mapper-attachments/licenses/commons-io-2.4.jar.sha1 b/plugins/mapper-attachments/licenses/commons-io-2.4.jar.sha1 new file mode 100644 index 00000000000..688318c938c --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-io-2.4.jar.sha1 @@ -0,0 +1 @@ +b1b6ea3b7e4aa4f492509a4952029cd8e48019ad diff --git a/plugins/mapper-attachments/licenses/commons-io-LICENSE.txt b/plugins/mapper-attachments/licenses/commons-io-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-io-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/mapper-attachments/licenses/commons-io-NOTICE.txt b/plugins/mapper-attachments/licenses/commons-io-NOTICE.txt new file mode 100644 index 00000000000..a6b77d1eb60 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-io-NOTICE.txt @@ -0,0 +1,5 @@ +Apache Commons IO +Copyright 2002-2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/mapper-attachments/licenses/commons-logging-1.1.3.jar.sha1 b/plugins/mapper-attachments/licenses/commons-logging-1.1.3.jar.sha1 new file mode 100644 index 00000000000..5b8f029e582 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-logging-1.1.3.jar.sha1 @@ -0,0 +1 @@ +f6f66e966c70a83ffbdb6f17a0919eaf7c8aca7f \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/commons-logging-LICENSE.txt b/plugins/mapper-attachments/licenses/commons-logging-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-logging-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/mapper-attachments/licenses/commons-logging-NOTICE.txt b/plugins/mapper-attachments/licenses/commons-logging-NOTICE.txt new file mode 100644 index 00000000000..d3d6e140ce4 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-logging-NOTICE.txt @@ -0,0 +1,5 @@ +Apache Commons Logging +Copyright 2003-2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/mapper-attachments/licenses/fontbox-1.8.10.jar.sha1 b/plugins/mapper-attachments/licenses/fontbox-1.8.10.jar.sha1 new file mode 100644 index 00000000000..ce7f9f5d49c --- /dev/null +++ b/plugins/mapper-attachments/licenses/fontbox-1.8.10.jar.sha1 @@ -0,0 +1 @@ +41776c7713e3f3a1ce688bd96459fc597298c340 diff --git a/plugins/mapper-attachments/licenses/fontbox-LICENSE.txt b/plugins/mapper-attachments/licenses/fontbox-LICENSE.txt new file mode 100644 index 00000000000..97553f24a43 --- /dev/null +++ b/plugins/mapper-attachments/licenses/fontbox-LICENSE.txt @@ -0,0 +1,344 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +EXTERNAL COMPONENTS + +Apache PDFBox includes a number of components with separate copyright notices +and license terms. Your use of these components is subject to the terms and +conditions of the following licenses. + +Contributions made to the original PDFBox and FontBox projects: + + Copyright (c) 2002-2007, www.pdfbox.org + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of pdfbox; nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + +Adobe Font Metrics (AFM) for PDF Core 14 Fonts + + This file and the 14 PostScript(R) AFM files it accompanies may be used, + copied, and distributed for any purpose and without charge, with or without + modification, provided that all copyright notices are retained; that the + AFM files are not distributed without this file; that all modifications + to this file or any of the AFM files are prominently noted in the modified + file(s); and that this paragraph is not modified. Adobe Systems has no + responsibility or obligation to support the use of the AFM files. + +CMaps for PDF Fonts (http://opensource.adobe.com/wiki/display/cmap/Downloads) + + Copyright 1990-2009 Adobe Systems Incorporated. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + Neither the name of Adobe Systems Incorporated nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + +PaDaF PDF/A preflight (http://sourceforge.net/projects/padaf) + + Copyright 2010 Atos Worldline SAS + + Licensed by Atos Worldline SAS under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + Atos Worldline SAS licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +OSXAdapter + + Version: 2.0 + + Disclaimer: IMPORTANT: This Apple software is supplied to you by + Apple Inc. ("Apple") in consideration of your agreement to the + following terms, and your use, installation, modification or + redistribution of this Apple software constitutes acceptance of these + terms. If you do not agree with these terms, please do not use, + install, modify or redistribute this Apple software. + + In consideration of your agreement to abide by the following terms, and + subject to these terms, Apple grants you a personal, non-exclusive + license, under Apple's copyrights in this original Apple software (the + "Apple Software"), to use, reproduce, modify and redistribute the Apple + Software, with or without modifications, in source and/or binary forms; + provided that if you redistribute the Apple Software in its entirety and + without modifications, you must retain this notice and the following + text and disclaimers in all such redistributions of the Apple Software. + Neither the name, trademarks, service marks or logos of Apple Inc. + may be used to endorse or promote products derived from the Apple + Software without specific prior written permission from Apple. Except + as expressly stated in this notice, no other rights or licenses, express + or implied, are granted by Apple herein, including but not limited to + any patent rights that may be infringed by your derivative works or by + other works in which the Apple Software may be incorporated. + + The Apple Software is provided by Apple on an "AS IS" basis. APPLE + MAKES NO WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION + THE IMPLIED WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE, REGARDING THE APPLE SOFTWARE OR ITS USE AND + OPERATION ALONE OR IN COMBINATION WITH YOUR PRODUCTS. + + IN NO EVENT SHALL APPLE BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) ARISING IN ANY WAY OUT OF THE USE, REPRODUCTION, + MODIFICATION AND/OR DISTRIBUTION OF THE APPLE SOFTWARE, HOWEVER CAUSED + AND WHETHER UNDER THEORY OF CONTRACT, TORT (INCLUDING NEGLIGENCE), + STRICT LIABILITY OR OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + Copyright (C) 2003-2007 Apple, Inc., All Rights Reserved diff --git a/plugins/mapper-attachments/licenses/fontbox-NOTICE.txt b/plugins/mapper-attachments/licenses/fontbox-NOTICE.txt new file mode 100644 index 00000000000..3c857082561 --- /dev/null +++ b/plugins/mapper-attachments/licenses/fontbox-NOTICE.txt @@ -0,0 +1,22 @@ +Apache PDFBox +Copyright 2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Based on source code originally developed in the PDFBox and +FontBox projects. + +Copyright (c) 2002-2007, www.pdfbox.org + +Based on source code originally developed in the PaDaF project. +Copyright (c) 2010 Atos Worldline SAS + +Includes the Adobe Glyph List +Copyright 1997, 1998, 2002, 2007, 2010 Adobe Systems Incorporated. + +Includes the Zapf Dingbats Glyph List +Copyright 2002, 2010 Adobe Systems Incorporated. + +Includes OSXAdapter +Copyright (C) 2003-2007 Apple, Inc., All Rights Reserved diff --git a/plugins/mapper-attachments/licenses/jempbox-1.8.10.jar.sha1 b/plugins/mapper-attachments/licenses/jempbox-1.8.10.jar.sha1 new file mode 100644 index 00000000000..5a7b1997208 --- /dev/null +++ b/plugins/mapper-attachments/licenses/jempbox-1.8.10.jar.sha1 @@ -0,0 +1 @@ +40df4e4ca884aadc20b82d5abd0a3679774c55a6 diff --git a/plugins/mapper-attachments/licenses/jempbox-LICENSE.txt b/plugins/mapper-attachments/licenses/jempbox-LICENSE.txt new file mode 100644 index 00000000000..1cf412f9c57 --- /dev/null +++ b/plugins/mapper-attachments/licenses/jempbox-LICENSE.txt @@ -0,0 +1,25 @@ +Copyright (c) 2006-2007, www.jempbox.org +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of fontbox; nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/plugins/mapper-attachments/licenses/jempbox-NOTICE.txt b/plugins/mapper-attachments/licenses/jempbox-NOTICE.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/licenses/juniversalchardet-1.0.3.jar.sha1 b/plugins/mapper-attachments/licenses/juniversalchardet-1.0.3.jar.sha1 new file mode 100644 index 00000000000..6b06952678f --- /dev/null +++ b/plugins/mapper-attachments/licenses/juniversalchardet-1.0.3.jar.sha1 @@ -0,0 +1 @@ +cd49678784c46aa8789c060538e0154013bb421b diff --git a/plugins/mapper-attachments/licenses/juniversalchardet-LICENSE.txt b/plugins/mapper-attachments/licenses/juniversalchardet-LICENSE.txt new file mode 100644 index 00000000000..06f965147a8 --- /dev/null +++ b/plugins/mapper-attachments/licenses/juniversalchardet-LICENSE.txt @@ -0,0 +1,469 @@ + MOZILLA PUBLIC LICENSE + Version 1.1 + + --------------- + +1. Definitions. + + 1.0.1. "Commercial Use" means distribution or otherwise making the + Covered Code available to a third party. + + 1.1. "Contributor" means each entity that creates or contributes to + the creation of Modifications. + + 1.2. "Contributor Version" means the combination of the Original + Code, prior Modifications used by a Contributor, and the Modifications + made by that particular Contributor. + + 1.3. "Covered Code" means the Original Code or Modifications or the + combination of the Original Code and Modifications, in each case + including portions thereof. + + 1.4. "Electronic Distribution Mechanism" means a mechanism generally + accepted in the software development community for the electronic + transfer of data. + + 1.5. "Executable" means Covered Code in any form other than Source + Code. + + 1.6. "Initial Developer" means the individual or entity identified + as the Initial Developer in the Source Code notice required by Exhibit + A. + + 1.7. "Larger Work" means a work which combines Covered Code or + portions thereof with code not governed by the terms of this License. + + 1.8. "License" means this document. + + 1.8.1. "Licensable" means having the right to grant, to the maximum + extent possible, whether at the time of the initial grant or + subsequently acquired, any and all of the rights conveyed herein. + + 1.9. "Modifications" means any addition to or deletion from the + substance or structure of either the Original Code or any previous + Modifications. When Covered Code is released as a series of files, a + Modification is: + A. Any addition to or deletion from the contents of a file + containing Original Code or previous Modifications. + + B. Any new file that contains any part of the Original Code or + previous Modifications. + + 1.10. "Original Code" means Source Code of computer software code + which is described in the Source Code notice required by Exhibit A as + Original Code, and which, at the time of its release under this + License is not already Covered Code governed by this License. + + 1.10.1. "Patent Claims" means any patent claim(s), now owned or + hereafter acquired, including without limitation, method, process, + and apparatus claims, in any patent Licensable by grantor. + + 1.11. "Source Code" means the preferred form of the Covered Code for + making modifications to it, including all modules it contains, plus + any associated interface definition files, scripts used to control + compilation and installation of an Executable, or source code + differential comparisons against either the Original Code or another + well known, available Covered Code of the Contributor's choice. The + Source Code can be in a compressed or archival form, provided the + appropriate decompression or de-archiving software is widely available + for no charge. + + 1.12. "You" (or "Your") means an individual or a legal entity + exercising rights under, and complying with all of the terms of, this + License or a future version of this License issued under Section 6.1. + For legal entities, "You" includes any entity which controls, is + controlled by, or is under common control with You. For purposes of + this definition, "control" means (a) the power, direct or indirect, + to cause the direction or management of such entity, whether by + contract or otherwise, or (b) ownership of more than fifty percent + (50%) of the outstanding shares or beneficial ownership of such + entity. + +2. Source Code License. + + 2.1. The Initial Developer Grant. + The Initial Developer hereby grants You a world-wide, royalty-free, + non-exclusive license, subject to third party intellectual property + claims: + (a) under intellectual property rights (other than patent or + trademark) Licensable by Initial Developer to use, reproduce, + modify, display, perform, sublicense and distribute the Original + Code (or portions thereof) with or without Modifications, and/or + as part of a Larger Work; and + + (b) under Patents Claims infringed by the making, using or + selling of Original Code, to make, have made, use, practice, + sell, and offer for sale, and/or otherwise dispose of the + Original Code (or portions thereof). + + (c) the licenses granted in this Section 2.1(a) and (b) are + effective on the date Initial Developer first distributes + Original Code under the terms of this License. + + (d) Notwithstanding Section 2.1(b) above, no patent license is + granted: 1) for code that You delete from the Original Code; 2) + separate from the Original Code; or 3) for infringements caused + by: i) the modification of the Original Code or ii) the + combination of the Original Code with other software or devices. + + 2.2. Contributor Grant. + Subject to third party intellectual property claims, each Contributor + hereby grants You a world-wide, royalty-free, non-exclusive license + + (a) under intellectual property rights (other than patent or + trademark) Licensable by Contributor, to use, reproduce, modify, + display, perform, sublicense and distribute the Modifications + created by such Contributor (or portions thereof) either on an + unmodified basis, with other Modifications, as Covered Code + and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using, or + selling of Modifications made by that Contributor either alone + and/or in combination with its Contributor Version (or portions + of such combination), to make, use, sell, offer for sale, have + made, and/or otherwise dispose of: 1) Modifications made by that + Contributor (or portions thereof); and 2) the combination of + Modifications made by that Contributor with its Contributor + Version (or portions of such combination). + + (c) the licenses granted in Sections 2.2(a) and 2.2(b) are + effective on the date Contributor first makes Commercial Use of + the Covered Code. + + (d) Notwithstanding Section 2.2(b) above, no patent license is + granted: 1) for any code that Contributor has deleted from the + Contributor Version; 2) separate from the Contributor Version; + 3) for infringements caused by: i) third party modifications of + Contributor Version or ii) the combination of Modifications made + by that Contributor with other software (except as part of the + Contributor Version) or other devices; or 4) under Patent Claims + infringed by Covered Code in the absence of Modifications made by + that Contributor. + +3. Distribution Obligations. + + 3.1. Application of License. + The Modifications which You create or to which You contribute are + governed by the terms of this License, including without limitation + Section 2.2. The Source Code version of Covered Code may be + distributed only under the terms of this License or a future version + of this License released under Section 6.1, and You must include a + copy of this License with every copy of the Source Code You + distribute. You may not offer or impose any terms on any Source Code + version that alters or restricts the applicable version of this + License or the recipients' rights hereunder. However, You may include + an additional document offering the additional rights described in + Section 3.5. + + 3.2. Availability of Source Code. + Any Modification which You create or to which You contribute must be + made available in Source Code form under the terms of this License + either on the same media as an Executable version or via an accepted + Electronic Distribution Mechanism to anyone to whom you made an + Executable version available; and if made available via Electronic + Distribution Mechanism, must remain available for at least twelve (12) + months after the date it initially became available, or at least six + (6) months after a subsequent version of that particular Modification + has been made available to such recipients. You are responsible for + ensuring that the Source Code version remains available even if the + Electronic Distribution Mechanism is maintained by a third party. + + 3.3. Description of Modifications. + You must cause all Covered Code to which You contribute to contain a + file documenting the changes You made to create that Covered Code and + the date of any change. You must include a prominent statement that + the Modification is derived, directly or indirectly, from Original + Code provided by the Initial Developer and including the name of the + Initial Developer in (a) the Source Code, and (b) in any notice in an + Executable version or related documentation in which You describe the + origin or ownership of the Covered Code. + + 3.4. Intellectual Property Matters + (a) Third Party Claims. + If Contributor has knowledge that a license under a third party's + intellectual property rights is required to exercise the rights + granted by such Contributor under Sections 2.1 or 2.2, + Contributor must include a text file with the Source Code + distribution titled "LEGAL" which describes the claim and the + party making the claim in sufficient detail that a recipient will + know whom to contact. If Contributor obtains such knowledge after + the Modification is made available as described in Section 3.2, + Contributor shall promptly modify the LEGAL file in all copies + Contributor makes available thereafter and shall take other steps + (such as notifying appropriate mailing lists or newsgroups) + reasonably calculated to inform those who received the Covered + Code that new knowledge has been obtained. + + (b) Contributor APIs. + If Contributor's Modifications include an application programming + interface and Contributor has knowledge of patent licenses which + are reasonably necessary to implement that API, Contributor must + also include this information in the LEGAL file. + + (c) Representations. + Contributor represents that, except as disclosed pursuant to + Section 3.4(a) above, Contributor believes that Contributor's + Modifications are Contributor's original creation(s) and/or + Contributor has sufficient rights to grant the rights conveyed by + this License. + + 3.5. Required Notices. + You must duplicate the notice in Exhibit A in each file of the Source + Code. If it is not possible to put such notice in a particular Source + Code file due to its structure, then You must include such notice in a + location (such as a relevant directory) where a user would be likely + to look for such a notice. If You created one or more Modification(s) + You may add your name as a Contributor to the notice described in + Exhibit A. You must also duplicate this License in any documentation + for the Source Code where You describe recipients' rights or ownership + rights relating to Covered Code. You may choose to offer, and to + charge a fee for, warranty, support, indemnity or liability + obligations to one or more recipients of Covered Code. However, You + may do so only on Your own behalf, and not on behalf of the Initial + Developer or any Contributor. You must make it absolutely clear than + any such warranty, support, indemnity or liability obligation is + offered by You alone, and You hereby agree to indemnify the Initial + Developer and every Contributor for any liability incurred by the + Initial Developer or such Contributor as a result of warranty, + support, indemnity or liability terms You offer. + + 3.6. Distribution of Executable Versions. + You may distribute Covered Code in Executable form only if the + requirements of Section 3.1-3.5 have been met for that Covered Code, + and if You include a notice stating that the Source Code version of + the Covered Code is available under the terms of this License, + including a description of how and where You have fulfilled the + obligations of Section 3.2. The notice must be conspicuously included + in any notice in an Executable version, related documentation or + collateral in which You describe recipients' rights relating to the + Covered Code. You may distribute the Executable version of Covered + Code or ownership rights under a license of Your choice, which may + contain terms different from this License, provided that You are in + compliance with the terms of this License and that the license for the + Executable version does not attempt to limit or alter the recipient's + rights in the Source Code version from the rights set forth in this + License. If You distribute the Executable version under a different + license You must make it absolutely clear that any terms which differ + from this License are offered by You alone, not by the Initial + Developer or any Contributor. You hereby agree to indemnify the + Initial Developer and every Contributor for any liability incurred by + the Initial Developer or such Contributor as a result of any such + terms You offer. + + 3.7. Larger Works. + You may create a Larger Work by combining Covered Code with other code + not governed by the terms of this License and distribute the Larger + Work as a single product. In such a case, You must make sure the + requirements of this License are fulfilled for the Covered Code. + +4. Inability to Comply Due to Statute or Regulation. + + If it is impossible for You to comply with any of the terms of this + License with respect to some or all of the Covered Code due to + statute, judicial order, or regulation then You must: (a) comply with + the terms of this License to the maximum extent possible; and (b) + describe the limitations and the code they affect. Such description + must be included in the LEGAL file described in Section 3.4 and must + be included with all distributions of the Source Code. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Application of this License. + + This License applies to code to which the Initial Developer has + attached the notice in Exhibit A and to related Covered Code. + +6. Versions of the License. + + 6.1. New Versions. + Netscape Communications Corporation ("Netscape") may publish revised + and/or new versions of the License from time to time. Each version + will be given a distinguishing version number. + + 6.2. Effect of New Versions. + Once Covered Code has been published under a particular version of the + License, You may always continue to use it under the terms of that + version. You may also choose to use such Covered Code under the terms + of any subsequent version of the License published by Netscape. No one + other than Netscape has the right to modify the terms applicable to + Covered Code created under this License. + + 6.3. Derivative Works. + If You create or use a modified version of this License (which you may + only do in order to apply it to code which is not already Covered Code + governed by this License), You must (a) rename Your license so that + the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape", + "MPL", "NPL" or any confusingly similar phrase do not appear in your + license (except to note that your license differs from this License) + and (b) otherwise make it clear that Your version of the license + contains terms which differ from the Mozilla Public License and + Netscape Public License. (Filling in the name of the Initial + Developer, Original Code or Contributor in the notice described in + Exhibit A shall not of themselves be deemed to be modifications of + this License.) + +7. DISCLAIMER OF WARRANTY. + + COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, + WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, + WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF + DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. + THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE + IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT, + YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE + COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER + OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF + ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +8. TERMINATION. + + 8.1. This License and the rights granted hereunder will terminate + automatically if You fail to comply with terms herein and fail to cure + such breach within 30 days of becoming aware of the breach. All + sublicenses to the Covered Code which are properly granted shall + survive any termination of this License. Provisions which, by their + nature, must remain in effect beyond the termination of this License + shall survive. + + 8.2. If You initiate litigation by asserting a patent infringement + claim (excluding declatory judgment actions) against Initial Developer + or a Contributor (the Initial Developer or Contributor against whom + You file such action is referred to as "Participant") alleging that: + + (a) such Participant's Contributor Version directly or indirectly + infringes any patent, then any and all rights granted by such + Participant to You under Sections 2.1 and/or 2.2 of this License + shall, upon 60 days notice from Participant terminate prospectively, + unless if within 60 days after receipt of notice You either: (i) + agree in writing to pay Participant a mutually agreeable reasonable + royalty for Your past and future use of Modifications made by such + Participant, or (ii) withdraw Your litigation claim with respect to + the Contributor Version against such Participant. If within 60 days + of notice, a reasonable royalty and payment arrangement are not + mutually agreed upon in writing by the parties or the litigation claim + is not withdrawn, the rights granted by Participant to You under + Sections 2.1 and/or 2.2 automatically terminate at the expiration of + the 60 day notice period specified above. + + (b) any software, hardware, or device, other than such Participant's + Contributor Version, directly or indirectly infringes any patent, then + any rights granted to You by such Participant under Sections 2.1(b) + and 2.2(b) are revoked effective as of the date You first made, used, + sold, distributed, or had made, Modifications made by that + Participant. + + 8.3. If You assert a patent infringement claim against Participant + alleging that such Participant's Contributor Version directly or + indirectly infringes any patent where such claim is resolved (such as + by license or settlement) prior to the initiation of patent + infringement litigation, then the reasonable value of the licenses + granted by such Participant under Sections 2.1 or 2.2 shall be taken + into account in determining the amount or value of any payment or + license. + + 8.4. In the event of termination under Sections 8.1 or 8.2 above, + all end user license agreements (excluding distributors and resellers) + which have been validly granted by You or any distributor hereunder + prior to termination shall survive termination. + +9. LIMITATION OF LIABILITY. + + UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT + (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL + DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE, + OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR + ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY + CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, + WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER + COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN + INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF + LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY + RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW + PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE + EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO + THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. + +10. U.S. GOVERNMENT END USERS. + + The Covered Code is a "commercial item," as that term is defined in + 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer + software" and "commercial computer software documentation," as such + terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 + C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), + all U.S. Government End Users acquire Covered Code with only those + rights set forth herein. + +11. MISCELLANEOUS. + + This License represents the complete agreement concerning subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. This License shall be governed by + California law provisions (except to the extent applicable law, if + any, provides otherwise), excluding its conflict-of-law provisions. + With respect to disputes in which at least one party is a citizen of, + or an entity chartered or registered to do business in the United + States of America, any litigation relating to this License shall be + subject to the jurisdiction of the Federal Courts of the Northern + District of California, with venue lying in Santa Clara County, + California, with the losing party responsible for costs, including + without limitation, court costs and reasonable attorneys' fees and + expenses. The application of the United Nations Convention on + Contracts for the International Sale of Goods is expressly excluded. + Any law or regulation which provides that the language of a contract + shall be construed against the drafter shall not apply to this + License. + +12. RESPONSIBILITY FOR CLAIMS. + + As between Initial Developer and the Contributors, each party is + responsible for claims and damages arising, directly or indirectly, + out of its utilization of rights under this License and You agree to + work with Initial Developer and Contributors to distribute such + responsibility on an equitable basis. Nothing herein is intended or + shall be deemed to constitute any admission of liability. + +13. MULTIPLE-LICENSED CODE. + + Initial Developer may designate portions of the Covered Code as + "Multiple-Licensed". "Multiple-Licensed" means that the Initial + Developer permits you to utilize portions of the Covered Code under + Your choice of the NPL or the alternative licenses, if any, specified + by the Initial Developer in the file described in Exhibit A. + +EXHIBIT A -Mozilla Public License. + + ``The contents of this file are subject to the Mozilla Public License + Version 1.1 (the "License"); you may not use this file except in + compliance with the License. You may obtain a copy of the License at + http://www.mozilla.org/MPL/ + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the + License for the specific language governing rights and limitations + under the License. + + The Original Code is ______________________________________. + + The Initial Developer of the Original Code is ________________________. + Portions created by ______________________ are Copyright (C) ______ + _______________________. All Rights Reserved. + + Contributor(s): ______________________________________. + + Alternatively, the contents of this file may be used under the terms + of the _____ license (the "[___] License"), in which case the + provisions of [______] License are applicable instead of those + above. If you wish to allow use of your version of this file only + under the terms of the [____] License and not to allow others to use + your version of this file under the MPL, indicate your decision by + deleting the provisions above and replace them with the notice and + other provisions required by the [___] License. If you do not delete + the provisions above, a recipient may use your version of this file + under either the MPL or the [___] License." + + [NOTE: The text of this Exhibit A may differ slightly from the text of + the notices in the Source Code files of the Original Code. You should + use the text of this Exhibit A rather than the text found in the + Original Code Source Code for Your Modifications.] diff --git a/plugins/mapper-attachments/licenses/juniversalchardet-NOTICE.txt b/plugins/mapper-attachments/licenses/juniversalchardet-NOTICE.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/licenses/pdfbox-1.8.10.jar.sha1 b/plugins/mapper-attachments/licenses/pdfbox-1.8.10.jar.sha1 new file mode 100644 index 00000000000..98ce1f9d98c --- /dev/null +++ b/plugins/mapper-attachments/licenses/pdfbox-1.8.10.jar.sha1 @@ -0,0 +1 @@ +bc5d1254495be36d0a3b3d6c35f88d05200b9311 diff --git a/plugins/mapper-attachments/licenses/pdfbox-LICENSE.txt b/plugins/mapper-attachments/licenses/pdfbox-LICENSE.txt new file mode 100644 index 00000000000..97553f24a43 --- /dev/null +++ b/plugins/mapper-attachments/licenses/pdfbox-LICENSE.txt @@ -0,0 +1,344 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +EXTERNAL COMPONENTS + +Apache PDFBox includes a number of components with separate copyright notices +and license terms. Your use of these components is subject to the terms and +conditions of the following licenses. + +Contributions made to the original PDFBox and FontBox projects: + + Copyright (c) 2002-2007, www.pdfbox.org + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of pdfbox; nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + +Adobe Font Metrics (AFM) for PDF Core 14 Fonts + + This file and the 14 PostScript(R) AFM files it accompanies may be used, + copied, and distributed for any purpose and without charge, with or without + modification, provided that all copyright notices are retained; that the + AFM files are not distributed without this file; that all modifications + to this file or any of the AFM files are prominently noted in the modified + file(s); and that this paragraph is not modified. Adobe Systems has no + responsibility or obligation to support the use of the AFM files. + +CMaps for PDF Fonts (http://opensource.adobe.com/wiki/display/cmap/Downloads) + + Copyright 1990-2009 Adobe Systems Incorporated. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + Neither the name of Adobe Systems Incorporated nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + +PaDaF PDF/A preflight (http://sourceforge.net/projects/padaf) + + Copyright 2010 Atos Worldline SAS + + Licensed by Atos Worldline SAS under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + Atos Worldline SAS licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +OSXAdapter + + Version: 2.0 + + Disclaimer: IMPORTANT: This Apple software is supplied to you by + Apple Inc. ("Apple") in consideration of your agreement to the + following terms, and your use, installation, modification or + redistribution of this Apple software constitutes acceptance of these + terms. If you do not agree with these terms, please do not use, + install, modify or redistribute this Apple software. + + In consideration of your agreement to abide by the following terms, and + subject to these terms, Apple grants you a personal, non-exclusive + license, under Apple's copyrights in this original Apple software (the + "Apple Software"), to use, reproduce, modify and redistribute the Apple + Software, with or without modifications, in source and/or binary forms; + provided that if you redistribute the Apple Software in its entirety and + without modifications, you must retain this notice and the following + text and disclaimers in all such redistributions of the Apple Software. + Neither the name, trademarks, service marks or logos of Apple Inc. + may be used to endorse or promote products derived from the Apple + Software without specific prior written permission from Apple. Except + as expressly stated in this notice, no other rights or licenses, express + or implied, are granted by Apple herein, including but not limited to + any patent rights that may be infringed by your derivative works or by + other works in which the Apple Software may be incorporated. + + The Apple Software is provided by Apple on an "AS IS" basis. APPLE + MAKES NO WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION + THE IMPLIED WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE, REGARDING THE APPLE SOFTWARE OR ITS USE AND + OPERATION ALONE OR IN COMBINATION WITH YOUR PRODUCTS. + + IN NO EVENT SHALL APPLE BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) ARISING IN ANY WAY OUT OF THE USE, REPRODUCTION, + MODIFICATION AND/OR DISTRIBUTION OF THE APPLE SOFTWARE, HOWEVER CAUSED + AND WHETHER UNDER THEORY OF CONTRACT, TORT (INCLUDING NEGLIGENCE), + STRICT LIABILITY OR OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + Copyright (C) 2003-2007 Apple, Inc., All Rights Reserved diff --git a/plugins/mapper-attachments/licenses/pdfbox-NOTICE.txt b/plugins/mapper-attachments/licenses/pdfbox-NOTICE.txt new file mode 100644 index 00000000000..3c857082561 --- /dev/null +++ b/plugins/mapper-attachments/licenses/pdfbox-NOTICE.txt @@ -0,0 +1,22 @@ +Apache PDFBox +Copyright 2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Based on source code originally developed in the PDFBox and +FontBox projects. + +Copyright (c) 2002-2007, www.pdfbox.org + +Based on source code originally developed in the PaDaF project. +Copyright (c) 2010 Atos Worldline SAS + +Includes the Adobe Glyph List +Copyright 1997, 1998, 2002, 2007, 2010 Adobe Systems Incorporated. + +Includes the Zapf Dingbats Glyph List +Copyright 2002, 2010 Adobe Systems Incorporated. + +Includes OSXAdapter +Copyright (C) 2003-2007 Apple, Inc., All Rights Reserved diff --git a/plugins/mapper-attachments/licenses/poi-3.13.jar.sha1 b/plugins/mapper-attachments/licenses/poi-3.13.jar.sha1 new file mode 100644 index 00000000000..09063c1e5e0 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-3.13.jar.sha1 @@ -0,0 +1 @@ +0f59f504ba8c521e61e25f417ec652fd485010f3 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/poi-LICENSE.txt b/plugins/mapper-attachments/licenses/poi-LICENSE.txt new file mode 100644 index 00000000000..dd2cbd5fbc1 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-LICENSE.txt @@ -0,0 +1,463 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +APACHE POI SUBCOMPONENTS: + +Apache POI includes subcomponents with separate copyright notices and +license terms. Your use of these subcomponents is subject to the terms +and conditions of the following licenses: + + +Office Open XML schemas (ooxml-schemas-1.1.jar) + + The Office Open XML schema definitions used by Apache POI are + a part of the Office Open XML ECMA Specification (ECMA-376, [1]). + As defined in section 9.4 of the ECMA bylaws [2], this specification + is available to all interested parties without restriction: + + 9.4 All documents when approved shall be made available to + all interested parties without restriction. + + Furthermore, both Microsoft and Adobe have granted patent licenses + to this work [3,4,5]. + + [1] http://www.ecma-international.org/publications/standards/Ecma-376.htm + [2] http://www.ecma-international.org/memento/Ecmabylaws.htm + [3] http://www.microsoft.com/openspecifications/en/us/programs/osp/default.aspx + [4] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Edition%202%20Microsoft%20Patent%20Declaration.pdf + [5] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Adobe%20Patent%20Declaration.pdf + + +JUnit test library (junit-4.11.jar) + + Common Public License - v 1.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + + 1. DEFINITIONS + + "Contribution" means: + + a) in the case of the initial Contributor, the initial code and + documentation distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + + i) changes to the Program, and + + ii) additions to the Program; + + where such changes and/or additions to the Program originate from + and are distributed by that particular Contributor. A Contribution + 'originates' from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include additions to the Program which: (i) are + separate modules of software distributed in conjunction with the + Program under their own license agreement, and (ii) are not derivative + works of the Program. + + "Contributor" means any person or entity that distributes the Program. + + "Licensed Patents " mean patent claims licensable by a Contributor which + are necessarily infringed by the use or sale of its Contribution alone + or when combined with the Program. + + "Program" means the Contributions distributed in accordance with this + Agreement. + + "Recipient" means anyone who receives the Program under this Agreement, + including all Contributors. + + 2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free copyright license + to reproduce, prepare derivative works of, publicly display, publicly + perform, distribute and sublicense the Contribution of such + Contributor, if any, and such derivative works, in source code and + object code form. + + b) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free patent license under + Licensed Patents to make, use, sell, offer to sell, import and + otherwise transfer the Contribution of such Contributor, if any, in + source code and object code form. This patent license shall apply to + the combination of the Contribution and the Program if, at the time + the Contribution is added by the Contributor, such addition of the + Contribution causes such combination to be covered by the Licensed + Patents. The patent license shall not apply to any other combinations + which include the Contribution. No hardware per se is licensed + hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the rights + and licenses granted hereunder, each Recipient hereby assumes sole + responsibility to secure any other intellectual property rights + needed, if any. For example, if a third party patent license is + required to allow Recipient to distribute the Program, it is + Recipient's responsibility to acquire that license before + distributing the Program. + + d) Each Contributor represents that to its knowledge it has sufficient + copyright rights in its Contribution, if any, to grant the copyright + license set forth in this Agreement. + + 3. REQUIREMENTS + + A Contributor may choose to distribute the Program in object code form + under its own license agreement, provided that: + + a) it complies with the terms and conditions of this Agreement; and + + b) its license agreement: + + i) effectively disclaims on behalf of all Contributors all warranties + and conditions, express and implied, including warranties or + conditions of title and non-infringement, and implied warranties + or conditions of merchantability and fitness for a particular + purpose; + + ii) effectively excludes on behalf of all Contributors all liability + for damages, including direct, indirect, special, incidental and + consequential damages, such as lost profits; + + iii) states that any provisions which differ from this Agreement are + offered by that Contributor alone and not by any other party; and + + iv) states that source code for the Program is available from such + Contributor, and informs licensees how to obtain it in a + reasonable manner on or through a medium customarily used for + software exchange. + + When the Program is made available in source code form: + + a) it must be made available under this Agreement; and + + b) a copy of this Agreement must be included with each copy of + the Program. + + Contributors may not remove or alter any copyright notices contained + within the Program. + + Each Contributor must identify itself as the originator of its + Contribution, if any, in a manner that reasonably allows subsequent + Recipients to identify the originator of the Contribution. + + 4. COMMERCIAL DISTRIBUTION + + Commercial distributors of software may accept certain responsibilities + with respect to end users, business partners and the like. While this + license is intended to facilitate the commercial use of the Program, + the Contributor who includes the Program in a commercial product offering + should do so in a manner which does not create potential liability for + other Contributors. Therefore, if a Contributor includes the Program + in a commercial product offering, such Contributor ("Commercial + Contributor") hereby agrees to defend and indemnify every other + Contributor ("Indemnified Contributor") against any losses, damages + and costs (collectively "Losses") arising from claims, lawsuits and + other legal actions brought by a third party against the Indemnified + Contributor to the extent caused by the acts or omissions of such + Commercial Contributor in connection with its distribution of the + Program in a commercial product offering. The obligations in this + section do not apply to any claims or Losses relating to any actual + or alleged intellectual property infringement. In order to qualify, + an Indemnified Contributor must: a) promptly notify the Commercial + Contributor in writing of such claim, and b) allow the Commercial + Contributor to control, and cooperate with the Commercial Contributor + in, the defense and any related settlement negotiations. The Indemnified + Contributor may participate in any such claim at its own expense. + + For example, a Contributor might include the Program in a commercial + product offering, Product X. That Contributor is then a Commercial + Contributor. If that Commercial Contributor then makes performance + claims, or offers warranties related to Product X, those performance + claims and warranties are such Commercial Contributor's responsibility + alone. Under this section, the Commercial Contributor would have to + defend claims against the other Contributors related to those + performance claims and warranties, and if a court requires any other + Contributor to pay any damages as a result, the Commercial Contributor + must pay those damages. + + 5. NO WARRANTY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED + ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER + EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR + CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR + A PARTICULAR PURPOSE. Each Recipient is solely responsible for + determining the appropriateness of using and distributing the Program + and assumes all risks associated with its exercise of rights under this + Agreement, including but not limited to the risks and costs of program + errors, compliance with applicable laws, damage to or loss of data, + programs or equipment, and unavailability or interruption of operations. + + 6. DISCLAIMER OF LIABILITY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR + ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING + WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR + DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED + HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 7. GENERAL + + If any provision of this Agreement is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this Agreement, and without further + action by the parties hereto, such provision shall be reformed to the + minimum extent necessary to make such provision valid and enforceable. + + If Recipient institutes patent litigation against a Contributor with + respect to a patent applicable to software (including a cross-claim or + counterclaim in a lawsuit), then any patent licenses granted by that + Contributor to such Recipient under this Agreement shall terminate as of + the date such litigation is filed. In addition, if Recipient institutes + patent litigation against any entity (including a cross-claim or + counterclaim in a lawsuit) alleging that the Program itself (excluding + combinations of the Program with other software or hardware) infringes + such Recipient's patent(s), then such Recipient's rights granted under + Section 2(b) shall terminate as of the date such litigation is filed. + + All Recipient's rights under this Agreement shall terminate if it fails + to comply with any of the material terms or conditions of this Agreement + and does not cure such failure in a reasonable period of time after + becoming aware of such noncompliance. If all Recipient's rights under + this Agreement terminate, Recipient agrees to cease use and distribution + of the Program as soon as reasonably practicable. However, Recipient's + obligations under this Agreement and any licenses granted by Recipient + relating to the Program shall continue and survive. + + Everyone is permitted to copy and distribute copies of this Agreement, + but in order to avoid inconsistency the Agreement is copyrighted and may + only be modified in the following manner. The Agreement Steward reserves + the right to publish new versions (including revisions) of this Agreement + from time to time. No one other than the Agreement Steward has the right + to modify this Agreement. IBM is the initial Agreement Steward. IBM may + assign the responsibility to serve as the Agreement Steward to a suitable + separate entity. Each new version of the Agreement will be given a + distinguishing version number. The Program (including Contributions) may + always be distributed subject to the version of the Agreement under which + it was received. In addition, after a new version of the Agreement is + published, Contributor may elect to distribute the Program (including + its Contributions) under the new version. Except as expressly stated in + Sections 2(a) and 2(b) above, Recipient receives no rights or licenses + to the intellectual property of any Contributor under this Agreement, + whether expressly, by implication, estoppel or otherwise. All rights in + the Program not expressly granted under this Agreement are reserved. + + This Agreement is governed by the laws of the State of New York and the + intellectual property laws of the United States of America. No party to + this Agreement will bring a legal action under this Agreement more than + one year after the cause of action arose. Each party waives its rights + to a jury trial in any resulting litigation. diff --git a/plugins/mapper-attachments/licenses/poi-NOTICE.txt b/plugins/mapper-attachments/licenses/poi-NOTICE.txt new file mode 100644 index 00000000000..12ff265290d --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-NOTICE.txt @@ -0,0 +1,23 @@ +Apache POI +Copyright 2003-2015 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +This product contains parts that were originally based on software from BEA. +Copyright (c) 2000-2003, BEA Systems, . + +This product contains W3C XML Schema documents. Copyright 2001-2003 (c) +World Wide Web Consortium (Massachusetts Institute of Technology, European +Research Consortium for Informatics and Mathematics, Keio University) + +This product contains the Piccolo XML Parser for Java +(http://piccolo.sourceforge.net/). Copyright 2002 Yuval Oren. + +This product contains the chunks_parse_cmds.tbl file from the vsdump program. +Copyright (C) 2006-2007 Valek Filippov (frob@df.ru) + +This product contains parts of the eID Applet project +(http://eid-applet.googlecode.com). Copyright (c) 2009-2014 +FedICT (federal ICT department of Belgium), e-Contract.be BVBA (https://www.e-contract.be), +Bart Hanssens from FedICT diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-3.13.jar.sha1 b/plugins/mapper-attachments/licenses/poi-ooxml-3.13.jar.sha1 new file mode 100644 index 00000000000..16784299855 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-ooxml-3.13.jar.sha1 @@ -0,0 +1 @@ +c364a8f5422d613e3a56db3b4b889f2989d7ee73 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-LICENSE.txt b/plugins/mapper-attachments/licenses/poi-ooxml-LICENSE.txt new file mode 100644 index 00000000000..dd2cbd5fbc1 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-ooxml-LICENSE.txt @@ -0,0 +1,463 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +APACHE POI SUBCOMPONENTS: + +Apache POI includes subcomponents with separate copyright notices and +license terms. Your use of these subcomponents is subject to the terms +and conditions of the following licenses: + + +Office Open XML schemas (ooxml-schemas-1.1.jar) + + The Office Open XML schema definitions used by Apache POI are + a part of the Office Open XML ECMA Specification (ECMA-376, [1]). + As defined in section 9.4 of the ECMA bylaws [2], this specification + is available to all interested parties without restriction: + + 9.4 All documents when approved shall be made available to + all interested parties without restriction. + + Furthermore, both Microsoft and Adobe have granted patent licenses + to this work [3,4,5]. + + [1] http://www.ecma-international.org/publications/standards/Ecma-376.htm + [2] http://www.ecma-international.org/memento/Ecmabylaws.htm + [3] http://www.microsoft.com/openspecifications/en/us/programs/osp/default.aspx + [4] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Edition%202%20Microsoft%20Patent%20Declaration.pdf + [5] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Adobe%20Patent%20Declaration.pdf + + +JUnit test library (junit-4.11.jar) + + Common Public License - v 1.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + + 1. DEFINITIONS + + "Contribution" means: + + a) in the case of the initial Contributor, the initial code and + documentation distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + + i) changes to the Program, and + + ii) additions to the Program; + + where such changes and/or additions to the Program originate from + and are distributed by that particular Contributor. A Contribution + 'originates' from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include additions to the Program which: (i) are + separate modules of software distributed in conjunction with the + Program under their own license agreement, and (ii) are not derivative + works of the Program. + + "Contributor" means any person or entity that distributes the Program. + + "Licensed Patents " mean patent claims licensable by a Contributor which + are necessarily infringed by the use or sale of its Contribution alone + or when combined with the Program. + + "Program" means the Contributions distributed in accordance with this + Agreement. + + "Recipient" means anyone who receives the Program under this Agreement, + including all Contributors. + + 2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free copyright license + to reproduce, prepare derivative works of, publicly display, publicly + perform, distribute and sublicense the Contribution of such + Contributor, if any, and such derivative works, in source code and + object code form. + + b) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free patent license under + Licensed Patents to make, use, sell, offer to sell, import and + otherwise transfer the Contribution of such Contributor, if any, in + source code and object code form. This patent license shall apply to + the combination of the Contribution and the Program if, at the time + the Contribution is added by the Contributor, such addition of the + Contribution causes such combination to be covered by the Licensed + Patents. The patent license shall not apply to any other combinations + which include the Contribution. No hardware per se is licensed + hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the rights + and licenses granted hereunder, each Recipient hereby assumes sole + responsibility to secure any other intellectual property rights + needed, if any. For example, if a third party patent license is + required to allow Recipient to distribute the Program, it is + Recipient's responsibility to acquire that license before + distributing the Program. + + d) Each Contributor represents that to its knowledge it has sufficient + copyright rights in its Contribution, if any, to grant the copyright + license set forth in this Agreement. + + 3. REQUIREMENTS + + A Contributor may choose to distribute the Program in object code form + under its own license agreement, provided that: + + a) it complies with the terms and conditions of this Agreement; and + + b) its license agreement: + + i) effectively disclaims on behalf of all Contributors all warranties + and conditions, express and implied, including warranties or + conditions of title and non-infringement, and implied warranties + or conditions of merchantability and fitness for a particular + purpose; + + ii) effectively excludes on behalf of all Contributors all liability + for damages, including direct, indirect, special, incidental and + consequential damages, such as lost profits; + + iii) states that any provisions which differ from this Agreement are + offered by that Contributor alone and not by any other party; and + + iv) states that source code for the Program is available from such + Contributor, and informs licensees how to obtain it in a + reasonable manner on or through a medium customarily used for + software exchange. + + When the Program is made available in source code form: + + a) it must be made available under this Agreement; and + + b) a copy of this Agreement must be included with each copy of + the Program. + + Contributors may not remove or alter any copyright notices contained + within the Program. + + Each Contributor must identify itself as the originator of its + Contribution, if any, in a manner that reasonably allows subsequent + Recipients to identify the originator of the Contribution. + + 4. COMMERCIAL DISTRIBUTION + + Commercial distributors of software may accept certain responsibilities + with respect to end users, business partners and the like. While this + license is intended to facilitate the commercial use of the Program, + the Contributor who includes the Program in a commercial product offering + should do so in a manner which does not create potential liability for + other Contributors. Therefore, if a Contributor includes the Program + in a commercial product offering, such Contributor ("Commercial + Contributor") hereby agrees to defend and indemnify every other + Contributor ("Indemnified Contributor") against any losses, damages + and costs (collectively "Losses") arising from claims, lawsuits and + other legal actions brought by a third party against the Indemnified + Contributor to the extent caused by the acts or omissions of such + Commercial Contributor in connection with its distribution of the + Program in a commercial product offering. The obligations in this + section do not apply to any claims or Losses relating to any actual + or alleged intellectual property infringement. In order to qualify, + an Indemnified Contributor must: a) promptly notify the Commercial + Contributor in writing of such claim, and b) allow the Commercial + Contributor to control, and cooperate with the Commercial Contributor + in, the defense and any related settlement negotiations. The Indemnified + Contributor may participate in any such claim at its own expense. + + For example, a Contributor might include the Program in a commercial + product offering, Product X. That Contributor is then a Commercial + Contributor. If that Commercial Contributor then makes performance + claims, or offers warranties related to Product X, those performance + claims and warranties are such Commercial Contributor's responsibility + alone. Under this section, the Commercial Contributor would have to + defend claims against the other Contributors related to those + performance claims and warranties, and if a court requires any other + Contributor to pay any damages as a result, the Commercial Contributor + must pay those damages. + + 5. NO WARRANTY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED + ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER + EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR + CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR + A PARTICULAR PURPOSE. Each Recipient is solely responsible for + determining the appropriateness of using and distributing the Program + and assumes all risks associated with its exercise of rights under this + Agreement, including but not limited to the risks and costs of program + errors, compliance with applicable laws, damage to or loss of data, + programs or equipment, and unavailability or interruption of operations. + + 6. DISCLAIMER OF LIABILITY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR + ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING + WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR + DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED + HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 7. GENERAL + + If any provision of this Agreement is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this Agreement, and without further + action by the parties hereto, such provision shall be reformed to the + minimum extent necessary to make such provision valid and enforceable. + + If Recipient institutes patent litigation against a Contributor with + respect to a patent applicable to software (including a cross-claim or + counterclaim in a lawsuit), then any patent licenses granted by that + Contributor to such Recipient under this Agreement shall terminate as of + the date such litigation is filed. In addition, if Recipient institutes + patent litigation against any entity (including a cross-claim or + counterclaim in a lawsuit) alleging that the Program itself (excluding + combinations of the Program with other software or hardware) infringes + such Recipient's patent(s), then such Recipient's rights granted under + Section 2(b) shall terminate as of the date such litigation is filed. + + All Recipient's rights under this Agreement shall terminate if it fails + to comply with any of the material terms or conditions of this Agreement + and does not cure such failure in a reasonable period of time after + becoming aware of such noncompliance. If all Recipient's rights under + this Agreement terminate, Recipient agrees to cease use and distribution + of the Program as soon as reasonably practicable. However, Recipient's + obligations under this Agreement and any licenses granted by Recipient + relating to the Program shall continue and survive. + + Everyone is permitted to copy and distribute copies of this Agreement, + but in order to avoid inconsistency the Agreement is copyrighted and may + only be modified in the following manner. The Agreement Steward reserves + the right to publish new versions (including revisions) of this Agreement + from time to time. No one other than the Agreement Steward has the right + to modify this Agreement. IBM is the initial Agreement Steward. IBM may + assign the responsibility to serve as the Agreement Steward to a suitable + separate entity. Each new version of the Agreement will be given a + distinguishing version number. The Program (including Contributions) may + always be distributed subject to the version of the Agreement under which + it was received. In addition, after a new version of the Agreement is + published, Contributor may elect to distribute the Program (including + its Contributions) under the new version. Except as expressly stated in + Sections 2(a) and 2(b) above, Recipient receives no rights or licenses + to the intellectual property of any Contributor under this Agreement, + whether expressly, by implication, estoppel or otherwise. All rights in + the Program not expressly granted under this Agreement are reserved. + + This Agreement is governed by the laws of the State of New York and the + intellectual property laws of the United States of America. No party to + this Agreement will bring a legal action under this Agreement more than + one year after the cause of action arose. Each party waives its rights + to a jury trial in any resulting litigation. diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-NOTICE.txt b/plugins/mapper-attachments/licenses/poi-ooxml-NOTICE.txt new file mode 100644 index 00000000000..12ff265290d --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-ooxml-NOTICE.txt @@ -0,0 +1,23 @@ +Apache POI +Copyright 2003-2015 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +This product contains parts that were originally based on software from BEA. +Copyright (c) 2000-2003, BEA Systems, . + +This product contains W3C XML Schema documents. Copyright 2001-2003 (c) +World Wide Web Consortium (Massachusetts Institute of Technology, European +Research Consortium for Informatics and Mathematics, Keio University) + +This product contains the Piccolo XML Parser for Java +(http://piccolo.sourceforge.net/). Copyright 2002 Yuval Oren. + +This product contains the chunks_parse_cmds.tbl file from the vsdump program. +Copyright (C) 2006-2007 Valek Filippov (frob@df.ru) + +This product contains parts of the eID Applet project +(http://eid-applet.googlecode.com). Copyright (c) 2009-2014 +FedICT (federal ICT department of Belgium), e-Contract.be BVBA (https://www.e-contract.be), +Bart Hanssens from FedICT diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.13.jar.sha1 b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.13.jar.sha1 new file mode 100644 index 00000000000..b5a3a05c489 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.13.jar.sha1 @@ -0,0 +1 @@ +56fb0b9f3ffc3d7f7fc9b59e17b5fa2c3ab921e7 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-schemas-LICENSE.txt b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-LICENSE.txt new file mode 100644 index 00000000000..dd2cbd5fbc1 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-LICENSE.txt @@ -0,0 +1,463 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +APACHE POI SUBCOMPONENTS: + +Apache POI includes subcomponents with separate copyright notices and +license terms. Your use of these subcomponents is subject to the terms +and conditions of the following licenses: + + +Office Open XML schemas (ooxml-schemas-1.1.jar) + + The Office Open XML schema definitions used by Apache POI are + a part of the Office Open XML ECMA Specification (ECMA-376, [1]). + As defined in section 9.4 of the ECMA bylaws [2], this specification + is available to all interested parties without restriction: + + 9.4 All documents when approved shall be made available to + all interested parties without restriction. + + Furthermore, both Microsoft and Adobe have granted patent licenses + to this work [3,4,5]. + + [1] http://www.ecma-international.org/publications/standards/Ecma-376.htm + [2] http://www.ecma-international.org/memento/Ecmabylaws.htm + [3] http://www.microsoft.com/openspecifications/en/us/programs/osp/default.aspx + [4] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Edition%202%20Microsoft%20Patent%20Declaration.pdf + [5] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Adobe%20Patent%20Declaration.pdf + + +JUnit test library (junit-4.11.jar) + + Common Public License - v 1.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + + 1. DEFINITIONS + + "Contribution" means: + + a) in the case of the initial Contributor, the initial code and + documentation distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + + i) changes to the Program, and + + ii) additions to the Program; + + where such changes and/or additions to the Program originate from + and are distributed by that particular Contributor. A Contribution + 'originates' from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include additions to the Program which: (i) are + separate modules of software distributed in conjunction with the + Program under their own license agreement, and (ii) are not derivative + works of the Program. + + "Contributor" means any person or entity that distributes the Program. + + "Licensed Patents " mean patent claims licensable by a Contributor which + are necessarily infringed by the use or sale of its Contribution alone + or when combined with the Program. + + "Program" means the Contributions distributed in accordance with this + Agreement. + + "Recipient" means anyone who receives the Program under this Agreement, + including all Contributors. + + 2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free copyright license + to reproduce, prepare derivative works of, publicly display, publicly + perform, distribute and sublicense the Contribution of such + Contributor, if any, and such derivative works, in source code and + object code form. + + b) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free patent license under + Licensed Patents to make, use, sell, offer to sell, import and + otherwise transfer the Contribution of such Contributor, if any, in + source code and object code form. This patent license shall apply to + the combination of the Contribution and the Program if, at the time + the Contribution is added by the Contributor, such addition of the + Contribution causes such combination to be covered by the Licensed + Patents. The patent license shall not apply to any other combinations + which include the Contribution. No hardware per se is licensed + hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the rights + and licenses granted hereunder, each Recipient hereby assumes sole + responsibility to secure any other intellectual property rights + needed, if any. For example, if a third party patent license is + required to allow Recipient to distribute the Program, it is + Recipient's responsibility to acquire that license before + distributing the Program. + + d) Each Contributor represents that to its knowledge it has sufficient + copyright rights in its Contribution, if any, to grant the copyright + license set forth in this Agreement. + + 3. REQUIREMENTS + + A Contributor may choose to distribute the Program in object code form + under its own license agreement, provided that: + + a) it complies with the terms and conditions of this Agreement; and + + b) its license agreement: + + i) effectively disclaims on behalf of all Contributors all warranties + and conditions, express and implied, including warranties or + conditions of title and non-infringement, and implied warranties + or conditions of merchantability and fitness for a particular + purpose; + + ii) effectively excludes on behalf of all Contributors all liability + for damages, including direct, indirect, special, incidental and + consequential damages, such as lost profits; + + iii) states that any provisions which differ from this Agreement are + offered by that Contributor alone and not by any other party; and + + iv) states that source code for the Program is available from such + Contributor, and informs licensees how to obtain it in a + reasonable manner on or through a medium customarily used for + software exchange. + + When the Program is made available in source code form: + + a) it must be made available under this Agreement; and + + b) a copy of this Agreement must be included with each copy of + the Program. + + Contributors may not remove or alter any copyright notices contained + within the Program. + + Each Contributor must identify itself as the originator of its + Contribution, if any, in a manner that reasonably allows subsequent + Recipients to identify the originator of the Contribution. + + 4. COMMERCIAL DISTRIBUTION + + Commercial distributors of software may accept certain responsibilities + with respect to end users, business partners and the like. While this + license is intended to facilitate the commercial use of the Program, + the Contributor who includes the Program in a commercial product offering + should do so in a manner which does not create potential liability for + other Contributors. Therefore, if a Contributor includes the Program + in a commercial product offering, such Contributor ("Commercial + Contributor") hereby agrees to defend and indemnify every other + Contributor ("Indemnified Contributor") against any losses, damages + and costs (collectively "Losses") arising from claims, lawsuits and + other legal actions brought by a third party against the Indemnified + Contributor to the extent caused by the acts or omissions of such + Commercial Contributor in connection with its distribution of the + Program in a commercial product offering. The obligations in this + section do not apply to any claims or Losses relating to any actual + or alleged intellectual property infringement. In order to qualify, + an Indemnified Contributor must: a) promptly notify the Commercial + Contributor in writing of such claim, and b) allow the Commercial + Contributor to control, and cooperate with the Commercial Contributor + in, the defense and any related settlement negotiations. The Indemnified + Contributor may participate in any such claim at its own expense. + + For example, a Contributor might include the Program in a commercial + product offering, Product X. That Contributor is then a Commercial + Contributor. If that Commercial Contributor then makes performance + claims, or offers warranties related to Product X, those performance + claims and warranties are such Commercial Contributor's responsibility + alone. Under this section, the Commercial Contributor would have to + defend claims against the other Contributors related to those + performance claims and warranties, and if a court requires any other + Contributor to pay any damages as a result, the Commercial Contributor + must pay those damages. + + 5. NO WARRANTY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED + ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER + EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR + CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR + A PARTICULAR PURPOSE. Each Recipient is solely responsible for + determining the appropriateness of using and distributing the Program + and assumes all risks associated with its exercise of rights under this + Agreement, including but not limited to the risks and costs of program + errors, compliance with applicable laws, damage to or loss of data, + programs or equipment, and unavailability or interruption of operations. + + 6. DISCLAIMER OF LIABILITY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR + ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING + WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR + DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED + HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 7. GENERAL + + If any provision of this Agreement is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this Agreement, and without further + action by the parties hereto, such provision shall be reformed to the + minimum extent necessary to make such provision valid and enforceable. + + If Recipient institutes patent litigation against a Contributor with + respect to a patent applicable to software (including a cross-claim or + counterclaim in a lawsuit), then any patent licenses granted by that + Contributor to such Recipient under this Agreement shall terminate as of + the date such litigation is filed. In addition, if Recipient institutes + patent litigation against any entity (including a cross-claim or + counterclaim in a lawsuit) alleging that the Program itself (excluding + combinations of the Program with other software or hardware) infringes + such Recipient's patent(s), then such Recipient's rights granted under + Section 2(b) shall terminate as of the date such litigation is filed. + + All Recipient's rights under this Agreement shall terminate if it fails + to comply with any of the material terms or conditions of this Agreement + and does not cure such failure in a reasonable period of time after + becoming aware of such noncompliance. If all Recipient's rights under + this Agreement terminate, Recipient agrees to cease use and distribution + of the Program as soon as reasonably practicable. However, Recipient's + obligations under this Agreement and any licenses granted by Recipient + relating to the Program shall continue and survive. + + Everyone is permitted to copy and distribute copies of this Agreement, + but in order to avoid inconsistency the Agreement is copyrighted and may + only be modified in the following manner. The Agreement Steward reserves + the right to publish new versions (including revisions) of this Agreement + from time to time. No one other than the Agreement Steward has the right + to modify this Agreement. IBM is the initial Agreement Steward. IBM may + assign the responsibility to serve as the Agreement Steward to a suitable + separate entity. Each new version of the Agreement will be given a + distinguishing version number. The Program (including Contributions) may + always be distributed subject to the version of the Agreement under which + it was received. In addition, after a new version of the Agreement is + published, Contributor may elect to distribute the Program (including + its Contributions) under the new version. Except as expressly stated in + Sections 2(a) and 2(b) above, Recipient receives no rights or licenses + to the intellectual property of any Contributor under this Agreement, + whether expressly, by implication, estoppel or otherwise. All rights in + the Program not expressly granted under this Agreement are reserved. + + This Agreement is governed by the laws of the State of New York and the + intellectual property laws of the United States of America. No party to + this Agreement will bring a legal action under this Agreement more than + one year after the cause of action arose. Each party waives its rights + to a jury trial in any resulting litigation. diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-schemas-NOTICE.txt b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-NOTICE.txt new file mode 100644 index 00000000000..12ff265290d --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-NOTICE.txt @@ -0,0 +1,23 @@ +Apache POI +Copyright 2003-2015 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +This product contains parts that were originally based on software from BEA. +Copyright (c) 2000-2003, BEA Systems, . + +This product contains W3C XML Schema documents. Copyright 2001-2003 (c) +World Wide Web Consortium (Massachusetts Institute of Technology, European +Research Consortium for Informatics and Mathematics, Keio University) + +This product contains the Piccolo XML Parser for Java +(http://piccolo.sourceforge.net/). Copyright 2002 Yuval Oren. + +This product contains the chunks_parse_cmds.tbl file from the vsdump program. +Copyright (C) 2006-2007 Valek Filippov (frob@df.ru) + +This product contains parts of the eID Applet project +(http://eid-applet.googlecode.com). Copyright (c) 2009-2014 +FedICT (federal ICT department of Belgium), e-Contract.be BVBA (https://www.e-contract.be), +Bart Hanssens from FedICT diff --git a/plugins/mapper-attachments/licenses/poi-scratchpad-3.13.jar.sha1 b/plugins/mapper-attachments/licenses/poi-scratchpad-3.13.jar.sha1 new file mode 100644 index 00000000000..cc61780e2a5 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-scratchpad-3.13.jar.sha1 @@ -0,0 +1 @@ +09d763275e6c7fa05d47e2581606748669e88c55 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/poi-scratchpad-LICENSE.txt b/plugins/mapper-attachments/licenses/poi-scratchpad-LICENSE.txt new file mode 100644 index 00000000000..dd2cbd5fbc1 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-scratchpad-LICENSE.txt @@ -0,0 +1,463 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +APACHE POI SUBCOMPONENTS: + +Apache POI includes subcomponents with separate copyright notices and +license terms. Your use of these subcomponents is subject to the terms +and conditions of the following licenses: + + +Office Open XML schemas (ooxml-schemas-1.1.jar) + + The Office Open XML schema definitions used by Apache POI are + a part of the Office Open XML ECMA Specification (ECMA-376, [1]). + As defined in section 9.4 of the ECMA bylaws [2], this specification + is available to all interested parties without restriction: + + 9.4 All documents when approved shall be made available to + all interested parties without restriction. + + Furthermore, both Microsoft and Adobe have granted patent licenses + to this work [3,4,5]. + + [1] http://www.ecma-international.org/publications/standards/Ecma-376.htm + [2] http://www.ecma-international.org/memento/Ecmabylaws.htm + [3] http://www.microsoft.com/openspecifications/en/us/programs/osp/default.aspx + [4] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Edition%202%20Microsoft%20Patent%20Declaration.pdf + [5] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Adobe%20Patent%20Declaration.pdf + + +JUnit test library (junit-4.11.jar) + + Common Public License - v 1.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + + 1. DEFINITIONS + + "Contribution" means: + + a) in the case of the initial Contributor, the initial code and + documentation distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + + i) changes to the Program, and + + ii) additions to the Program; + + where such changes and/or additions to the Program originate from + and are distributed by that particular Contributor. A Contribution + 'originates' from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include additions to the Program which: (i) are + separate modules of software distributed in conjunction with the + Program under their own license agreement, and (ii) are not derivative + works of the Program. + + "Contributor" means any person or entity that distributes the Program. + + "Licensed Patents " mean patent claims licensable by a Contributor which + are necessarily infringed by the use or sale of its Contribution alone + or when combined with the Program. + + "Program" means the Contributions distributed in accordance with this + Agreement. + + "Recipient" means anyone who receives the Program under this Agreement, + including all Contributors. + + 2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free copyright license + to reproduce, prepare derivative works of, publicly display, publicly + perform, distribute and sublicense the Contribution of such + Contributor, if any, and such derivative works, in source code and + object code form. + + b) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free patent license under + Licensed Patents to make, use, sell, offer to sell, import and + otherwise transfer the Contribution of such Contributor, if any, in + source code and object code form. This patent license shall apply to + the combination of the Contribution and the Program if, at the time + the Contribution is added by the Contributor, such addition of the + Contribution causes such combination to be covered by the Licensed + Patents. The patent license shall not apply to any other combinations + which include the Contribution. No hardware per se is licensed + hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the rights + and licenses granted hereunder, each Recipient hereby assumes sole + responsibility to secure any other intellectual property rights + needed, if any. For example, if a third party patent license is + required to allow Recipient to distribute the Program, it is + Recipient's responsibility to acquire that license before + distributing the Program. + + d) Each Contributor represents that to its knowledge it has sufficient + copyright rights in its Contribution, if any, to grant the copyright + license set forth in this Agreement. + + 3. REQUIREMENTS + + A Contributor may choose to distribute the Program in object code form + under its own license agreement, provided that: + + a) it complies with the terms and conditions of this Agreement; and + + b) its license agreement: + + i) effectively disclaims on behalf of all Contributors all warranties + and conditions, express and implied, including warranties or + conditions of title and non-infringement, and implied warranties + or conditions of merchantability and fitness for a particular + purpose; + + ii) effectively excludes on behalf of all Contributors all liability + for damages, including direct, indirect, special, incidental and + consequential damages, such as lost profits; + + iii) states that any provisions which differ from this Agreement are + offered by that Contributor alone and not by any other party; and + + iv) states that source code for the Program is available from such + Contributor, and informs licensees how to obtain it in a + reasonable manner on or through a medium customarily used for + software exchange. + + When the Program is made available in source code form: + + a) it must be made available under this Agreement; and + + b) a copy of this Agreement must be included with each copy of + the Program. + + Contributors may not remove or alter any copyright notices contained + within the Program. + + Each Contributor must identify itself as the originator of its + Contribution, if any, in a manner that reasonably allows subsequent + Recipients to identify the originator of the Contribution. + + 4. COMMERCIAL DISTRIBUTION + + Commercial distributors of software may accept certain responsibilities + with respect to end users, business partners and the like. While this + license is intended to facilitate the commercial use of the Program, + the Contributor who includes the Program in a commercial product offering + should do so in a manner which does not create potential liability for + other Contributors. Therefore, if a Contributor includes the Program + in a commercial product offering, such Contributor ("Commercial + Contributor") hereby agrees to defend and indemnify every other + Contributor ("Indemnified Contributor") against any losses, damages + and costs (collectively "Losses") arising from claims, lawsuits and + other legal actions brought by a third party against the Indemnified + Contributor to the extent caused by the acts or omissions of such + Commercial Contributor in connection with its distribution of the + Program in a commercial product offering. The obligations in this + section do not apply to any claims or Losses relating to any actual + or alleged intellectual property infringement. In order to qualify, + an Indemnified Contributor must: a) promptly notify the Commercial + Contributor in writing of such claim, and b) allow the Commercial + Contributor to control, and cooperate with the Commercial Contributor + in, the defense and any related settlement negotiations. The Indemnified + Contributor may participate in any such claim at its own expense. + + For example, a Contributor might include the Program in a commercial + product offering, Product X. That Contributor is then a Commercial + Contributor. If that Commercial Contributor then makes performance + claims, or offers warranties related to Product X, those performance + claims and warranties are such Commercial Contributor's responsibility + alone. Under this section, the Commercial Contributor would have to + defend claims against the other Contributors related to those + performance claims and warranties, and if a court requires any other + Contributor to pay any damages as a result, the Commercial Contributor + must pay those damages. + + 5. NO WARRANTY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED + ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER + EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR + CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR + A PARTICULAR PURPOSE. Each Recipient is solely responsible for + determining the appropriateness of using and distributing the Program + and assumes all risks associated with its exercise of rights under this + Agreement, including but not limited to the risks and costs of program + errors, compliance with applicable laws, damage to or loss of data, + programs or equipment, and unavailability or interruption of operations. + + 6. DISCLAIMER OF LIABILITY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR + ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING + WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR + DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED + HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 7. GENERAL + + If any provision of this Agreement is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this Agreement, and without further + action by the parties hereto, such provision shall be reformed to the + minimum extent necessary to make such provision valid and enforceable. + + If Recipient institutes patent litigation against a Contributor with + respect to a patent applicable to software (including a cross-claim or + counterclaim in a lawsuit), then any patent licenses granted by that + Contributor to such Recipient under this Agreement shall terminate as of + the date such litigation is filed. In addition, if Recipient institutes + patent litigation against any entity (including a cross-claim or + counterclaim in a lawsuit) alleging that the Program itself (excluding + combinations of the Program with other software or hardware) infringes + such Recipient's patent(s), then such Recipient's rights granted under + Section 2(b) shall terminate as of the date such litigation is filed. + + All Recipient's rights under this Agreement shall terminate if it fails + to comply with any of the material terms or conditions of this Agreement + and does not cure such failure in a reasonable period of time after + becoming aware of such noncompliance. If all Recipient's rights under + this Agreement terminate, Recipient agrees to cease use and distribution + of the Program as soon as reasonably practicable. However, Recipient's + obligations under this Agreement and any licenses granted by Recipient + relating to the Program shall continue and survive. + + Everyone is permitted to copy and distribute copies of this Agreement, + but in order to avoid inconsistency the Agreement is copyrighted and may + only be modified in the following manner. The Agreement Steward reserves + the right to publish new versions (including revisions) of this Agreement + from time to time. No one other than the Agreement Steward has the right + to modify this Agreement. IBM is the initial Agreement Steward. IBM may + assign the responsibility to serve as the Agreement Steward to a suitable + separate entity. Each new version of the Agreement will be given a + distinguishing version number. The Program (including Contributions) may + always be distributed subject to the version of the Agreement under which + it was received. In addition, after a new version of the Agreement is + published, Contributor may elect to distribute the Program (including + its Contributions) under the new version. Except as expressly stated in + Sections 2(a) and 2(b) above, Recipient receives no rights or licenses + to the intellectual property of any Contributor under this Agreement, + whether expressly, by implication, estoppel or otherwise. All rights in + the Program not expressly granted under this Agreement are reserved. + + This Agreement is governed by the laws of the State of New York and the + intellectual property laws of the United States of America. No party to + this Agreement will bring a legal action under this Agreement more than + one year after the cause of action arose. Each party waives its rights + to a jury trial in any resulting litigation. diff --git a/plugins/mapper-attachments/licenses/poi-scratchpad-NOTICE.txt b/plugins/mapper-attachments/licenses/poi-scratchpad-NOTICE.txt new file mode 100644 index 00000000000..12ff265290d --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-scratchpad-NOTICE.txt @@ -0,0 +1,23 @@ +Apache POI +Copyright 2003-2015 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +This product contains parts that were originally based on software from BEA. +Copyright (c) 2000-2003, BEA Systems, . + +This product contains W3C XML Schema documents. Copyright 2001-2003 (c) +World Wide Web Consortium (Massachusetts Institute of Technology, European +Research Consortium for Informatics and Mathematics, Keio University) + +This product contains the Piccolo XML Parser for Java +(http://piccolo.sourceforge.net/). Copyright 2002 Yuval Oren. + +This product contains the chunks_parse_cmds.tbl file from the vsdump program. +Copyright (C) 2006-2007 Valek Filippov (frob@df.ru) + +This product contains parts of the eID Applet project +(http://eid-applet.googlecode.com). Copyright (c) 2009-2014 +FedICT (federal ICT department of Belgium), e-Contract.be BVBA (https://www.e-contract.be), +Bart Hanssens from FedICT diff --git a/plugins/mapper-attachments/licenses/stax-api-1.0.1.jar.sha1 b/plugins/mapper-attachments/licenses/stax-api-1.0.1.jar.sha1 new file mode 100644 index 00000000000..4426e34685d --- /dev/null +++ b/plugins/mapper-attachments/licenses/stax-api-1.0.1.jar.sha1 @@ -0,0 +1 @@ +49c100caf72d658aca8e58bd74a4ba90fa2b0d70 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/stax-api-LICENSE.txt b/plugins/mapper-attachments/licenses/stax-api-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/plugins/mapper-attachments/licenses/stax-api-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/mapper-attachments/licenses/stax-api-NOTICE.txt b/plugins/mapper-attachments/licenses/stax-api-NOTICE.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/licenses/tagsoup-1.2.1.jar.sha1 b/plugins/mapper-attachments/licenses/tagsoup-1.2.1.jar.sha1 new file mode 100644 index 00000000000..5d227b11a0f --- /dev/null +++ b/plugins/mapper-attachments/licenses/tagsoup-1.2.1.jar.sha1 @@ -0,0 +1 @@ +5584627487e984c03456266d3f8802eb85a9ce97 diff --git a/plugins/mapper-attachments/licenses/tagsoup-LICENSE.txt b/plugins/mapper-attachments/licenses/tagsoup-LICENSE.txt new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/plugins/mapper-attachments/licenses/tagsoup-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/mapper-attachments/licenses/tagsoup-NOTICE.txt b/plugins/mapper-attachments/licenses/tagsoup-NOTICE.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/licenses/tika-core-1.11.jar.sha1 b/plugins/mapper-attachments/licenses/tika-core-1.11.jar.sha1 new file mode 100644 index 00000000000..a6dfd778a9c --- /dev/null +++ b/plugins/mapper-attachments/licenses/tika-core-1.11.jar.sha1 @@ -0,0 +1 @@ +d37a6b9080c8361e47b2050f69833fd61501ede9 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/tika-core-LICENSE.txt b/plugins/mapper-attachments/licenses/tika-core-LICENSE.txt new file mode 100644 index 00000000000..9537d733ea9 --- /dev/null +++ b/plugins/mapper-attachments/licenses/tika-core-LICENSE.txt @@ -0,0 +1,372 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + +APACHE TIKA SUBCOMPONENTS + +Apache Tika includes a number of subcomponents with separate copyright notices +and license terms. Your use of these subcomponents is subject to the terms and +conditions of the following licenses. + +MIME type information from file-4.26.tar.gz (http://www.darwinsys.com/file/) + + Copyright (c) Ian F. Darwin 1986, 1987, 1989, 1990, 1991, 1992, 1994, 1995. + Software written by Ian F. Darwin and others; + maintained 1994- Christos Zoulas. + + This software is not subject to any export provision of the United States + Department of Commerce, and may be exported to any country or planet. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice immediately at the beginning of the file, without modification, + this list of conditions, and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + +Charset detection code from ICU4J (http://site.icu-project.org/) + + Copyright (c) 1995-2009 International Business Machines Corporation + and others + + All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, and/or sell copies of the Software, and to permit persons + to whom the Software is furnished to do so, provided that the above + copyright notice(s) and this permission notice appear in all copies + of the Software and that both the above copyright notice(s) and this + permission notice appear in supporting documentation. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. + IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE + BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, + OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, + WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, + ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS + SOFTWARE. + + Except as contained in this notice, the name of a copyright holder shall + not be used in advertising or otherwise to promote the sale, use or other + dealings in this Software without prior written authorization of the + copyright holder. + + +Parsing functionality provided by the NetCDF Java Library (http://www.unidata.ucar.edu/software/netcdf-java/) + + Copyright 1993-2010 University Corporation for Atmospheric Research/Unidata + + Portions of this software were developed by the Unidata Program at the University + Corporation for Atmospheric Research. + + Access and use of this software shall impose the following obligations and understandings + on the user. The user is granted the right, without any fee or cost, to use, copy, modify, + alter, enhance and distribute this software, and any derivative works thereof, and its + supporting documentation for any purpose whatsoever, provided that this entire notice + appears in all copies of the software, derivative works and supporting documentation. Further, + UCAR requests that the user credit UCAR/Unidata in any publications that result from the use + of this software or in any product that includes this software, although this is not an obligation. + The names UCAR and/or Unidata, however, may not be used in any advertising or publicity to endorse + or promote any products or commercial entity unless specific written permission is obtained from + UCAR/Unidata. The user also understands that UCAR/Unidata is not obligated to provide the user with + any support, consulting, training or assistance of any kind with regard to the use, operation and + performance of this software nor to provide the user with any updates, revisions, new versions or + "bug fixes." + + THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, + BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE ACCESS, + USE OR PERFORMANCE OF THIS SOFTWARE. + + +IPTC Photo Metadata descriptions are taken from the IPTC Photo Metadata +Standard, July 2010, Copyright 2010 International Press Telecommunications +Council. + + 1. The Specifications and Materials are licensed for use only on the condition that you agree to be bound by the terms of this license. Subject to this and other licensing requirements contained herein, you may, on a non-exclusive basis, use the Specifications and Materials. + 2. The IPTC openly provides the Specifications and Materials for voluntary use by individuals, partnerships, companies, corporations, organizations and any other entity for use at the entity's own risk. This disclaimer, license and release is intended to apply to the IPTC, its officers, directors, agents, representatives, members, contributors, affiliates, contractors, or co-venturers acting jointly or severally. + 3. The Document and translations thereof may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the copyright and license notices and references to the IPTC appearing in the Document and the terms of this Specifications License Agreement are included on all such copies and derivative works. Further, upon the receipt of written permission from the IPTC, the Document may be modified for the purpose of developing applications that use IPTC Specifications or as required to translate the Document into languages other than English. + 4. Any use, duplication, distribution, or exploitation of the Document and Specifications and Materials in any manner is at your own risk. + 5. NO WARRANTY, EXPRESSED OR IMPLIED, IS MADE REGARDING THE ACCURACY, ADEQUACY, COMPLETENESS, LEGALITY, RELIABILITY OR USEFULNESS OF ANY INFORMATION CONTAINED IN THE DOCUMENT OR IN ANY SPECIFICATION OR OTHER PRODUCT OR SERVICE PRODUCED OR SPONSORED BY THE IPTC. THE DOCUMENT AND THE INFORMATION CONTAINED HEREIN AND INCLUDED IN ANY SPECIFICATION OR OTHER PRODUCT OR SERVICE OF THE IPTC IS PROVIDED ON AN "AS IS" BASIS. THE IPTC DISCLAIMS ALL WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, ANY ACTUAL OR ASSERTED WARRANTY OF NON-INFRINGEMENT OF PROPRIETARY RIGHTS, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. NEITHER THE IPTC NOR ITS CONTRIBUTORS SHALL BE HELD LIABLE FOR ANY IMPROPER OR INCORRECT USE OF INFORMATION. NEITHER THE IPTC NOR ITS CONTRIBUTORS ASSUME ANY RESPONSIBILITY FOR ANYONE'S USE OF INFORMATION PROVIDED BY THE IPTC. IN NO EVENT SHALL THE IPTC OR ITS CONTRIBUTORS BE LIABLE TO ANYONE FOR DAMAGES OF ANY KIND, INCLUDING BUT NOT LIMITED TO, COMPENSATORY DAMAGES, LOST PROFITS, LOST DATA OR ANY FORM OF SPECIAL, INCIDENTAL, INDIRECT, CONSEQUENTIAL OR PUNITIVE DAMAGES OF ANY KIND WHETHER BASED ON BREACH OF CONTRACT OR WARRANTY, TORT, PRODUCT LIABILITY OR OTHERWISE. + 6. The IPTC takes no position regarding the validity or scope of any Intellectual Property or other rights that might be claimed to pertain to the implementation or use of the technology described in the Document or the extent to which any license under such rights might or might not be available. The IPTC does not represent that it has made any effort to identify any such rights. Copies of claims of rights made available for publication, assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of the Specifications and Materials, can be obtained from the Managing Director of the IPTC. + 7. By using the Specifications and Materials including the Document in any manner or for any purpose, you release the IPTC from all liabilities, claims, causes of action, allegations, losses, injuries, damages, or detriments of any nature arising from or relating to the use of the Specifications, Materials or any portion thereof. You further agree not to file a lawsuit, make a claim, or take any other formal or informal legal action against the IPTC, resulting from your acquisition, use, duplication, distribution, or exploitation of the Specifications, Materials or any portion thereof. Finally, you hereby agree that the IPTC is not liable for any direct, indirect, special or consequential damages arising from or relating to your acquisition, use, duplication, distribution, or exploitation of the Specifications, Materials or any portion thereof. + 8. Specifications and Materials may be downloaded or copied provided that ALL copies retain the ownership, copyright and license notices. + 9. Materials may not be edited, modified, or presented in a context that creates a misleading or false impression or statement as to the positions, actions, or statements of the IPTC. + 10. The name and trademarks of the IPTC may not be used in advertising, publicity, or in relation to products or services and their names without the specific, written prior permission of the IPTC. Any permitted use of the trademarks of the IPTC, whether registered or not, shall be accompanied by an appropriate mark and attribution, as agreed with the IPTC. + 11. Specifications may be extended by both members and non-members to provide additional functionality (Extension Specifications) provided that there is a clear recognition of the IPTC IP and its ownership in the Extension Specifications and the related documentation and provided that the extensions are clearly identified and provided that a perpetual license is granted by the creator of the Extension Specifications for other members and non-members to use the Extension Specifications and to continue extensions of the Extension Specifications. The IPTC does not waive any of its rights in the Specifications and Materials in this context. The Extension Specifications may be considered the intellectual property of their creator. The IPTC expressly disclaims any responsibility for damage caused by an extension to the Specifications. + 12. Specifications and Materials may be included in derivative work of both members and non-members provided that there is a clear recognition of the IPTC IP and its ownership in the derivative work and its related documentation. The IPTC does not waive any of its rights in the Specifications and Materials in this context. Derivative work in its entirety may be considered the intellectual property of the creator of the work .The IPTC expressly disclaims any responsibility for damage caused when its IP is used in a derivative context. + 13. This Specifications License Agreement is perpetual subject to your conformance to the terms of this Agreement. The IPTC may terminate this Specifications License Agreement immediately upon your breach of this Agreement and, upon such termination you will cease all use, duplication, distribution, and/or exploitation in any manner of the Specifications and Materials. + 14. This Specifications License Agreement reflects the entire agreement of the parties regarding the subject matter hereof and supersedes all prior agreements or representations regarding such matters, whether written or oral. To the extent any portion or provision of this Specifications License Agreement is found to be illegal or unenforceable, then the remaining provisions of this Specifications License Agreement will remain in full force and effect and the illegal or unenforceable provision will be construed to give it such effect as it may properly have that is consistent with the intentions of the parties. + 15. This Specifications License Agreement may only be modified in writing signed by an authorized representative of the IPTC. + 16. This Specifications License Agreement is governed by the law of United Kingdom, as such law is applied to contracts made and fully performed in the United Kingdom. Any disputes arising from or relating to this Specifications License Agreement will be resolved in the courts of the United Kingdom. You consent to the jurisdiction of such courts over you and covenant not to assert before such courts any objection to proceeding in such forums. + + +JUnRAR (https://github.com/edmund-wagner/junrar/) + + JUnRAR is based on the UnRAR tool, and covered by the same license + It was formerly available from http://java-unrar.svn.sourceforge.net/ + + ****** ***** ****** UnRAR - free utility for RAR archives + ** ** ** ** ** ** ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ****** ******* ****** License for use and distribution of + ** ** ** ** ** ** ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ** ** ** ** ** ** FREE portable version + ~~~~~~~~~~~~~~~~~~~~~ + + The source code of UnRAR utility is freeware. This means: + + 1. All copyrights to RAR and the utility UnRAR are exclusively + owned by the author - Alexander Roshal. + + 2. The UnRAR sources may be used in any software to handle RAR + archives without limitations free of charge, but cannot be used + to re-create the RAR compression algorithm, which is proprietary. + Distribution of modified UnRAR sources in separate form or as a + part of other software is permitted, provided that it is clearly + stated in the documentation and source comments that the code may + not be used to develop a RAR (WinRAR) compatible archiver. + + 3. The UnRAR utility may be freely distributed. It is allowed + to distribute UnRAR inside of other software packages. + + 4. THE RAR ARCHIVER AND THE UnRAR UTILITY ARE DISTRIBUTED "AS IS". + NO WARRANTY OF ANY KIND IS EXPRESSED OR IMPLIED. YOU USE AT + YOUR OWN RISK. THE AUTHOR WILL NOT BE LIABLE FOR DATA LOSS, + DAMAGES, LOSS OF PROFITS OR ANY OTHER KIND OF LOSS WHILE USING + OR MISUSING THIS SOFTWARE. + + 5. Installing and using the UnRAR utility signifies acceptance of + these terms and conditions of the license. + + 6. If you don't agree with terms of the license you must remove + UnRAR files from your storage devices and cease to use the + utility. + + Thank you for your interest in RAR and UnRAR. Alexander L. Roshal + +Sqlite (bundled in org.xerial's sqlite-jdbc) + This product bundles Sqlite, which is in the Public Domain. For details + see: https://www.sqlite.org/copyright.html diff --git a/plugins/mapper-attachments/licenses/tika-core-NOTICE.txt b/plugins/mapper-attachments/licenses/tika-core-NOTICE.txt new file mode 100644 index 00000000000..8e94f644b81 --- /dev/null +++ b/plugins/mapper-attachments/licenses/tika-core-NOTICE.txt @@ -0,0 +1,17 @@ +Apache Tika +Copyright 2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Copyright 1993-2010 University Corporation for Atmospheric Research/Unidata +This software contains code derived from UCAR/Unidata's NetCDF library. + +Tika-server component uses CDDL-licensed dependencies: jersey (http://jersey.java.net/) and +Grizzly (http://grizzly.java.net/) + +Tika-parsers component uses CDDL/LGPL dual-licensed dependency: jhighlight (https://github.com/codelibs/jhighlight) + +OpenCSV: Copyright 2005 Bytecode Pty Ltd. Licensed under the Apache License, Version 2.0 + +IPTC Photo Metadata descriptions Copyright 2010 International Press Telecommunications Council. diff --git a/plugins/mapper-attachments/licenses/tika-parsers-1.11.jar.sha1 b/plugins/mapper-attachments/licenses/tika-parsers-1.11.jar.sha1 new file mode 100644 index 00000000000..fbbd59efaf9 --- /dev/null +++ b/plugins/mapper-attachments/licenses/tika-parsers-1.11.jar.sha1 @@ -0,0 +1 @@ +355dc05d842ed223fc682da472229473ba706d68 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/tika-parsers-LICENSE.txt b/plugins/mapper-attachments/licenses/tika-parsers-LICENSE.txt new file mode 100644 index 00000000000..9537d733ea9 --- /dev/null +++ b/plugins/mapper-attachments/licenses/tika-parsers-LICENSE.txt @@ -0,0 +1,372 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + +APACHE TIKA SUBCOMPONENTS + +Apache Tika includes a number of subcomponents with separate copyright notices +and license terms. Your use of these subcomponents is subject to the terms and +conditions of the following licenses. + +MIME type information from file-4.26.tar.gz (http://www.darwinsys.com/file/) + + Copyright (c) Ian F. Darwin 1986, 1987, 1989, 1990, 1991, 1992, 1994, 1995. + Software written by Ian F. Darwin and others; + maintained 1994- Christos Zoulas. + + This software is not subject to any export provision of the United States + Department of Commerce, and may be exported to any country or planet. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice immediately at the beginning of the file, without modification, + this list of conditions, and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + +Charset detection code from ICU4J (http://site.icu-project.org/) + + Copyright (c) 1995-2009 International Business Machines Corporation + and others + + All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, and/or sell copies of the Software, and to permit persons + to whom the Software is furnished to do so, provided that the above + copyright notice(s) and this permission notice appear in all copies + of the Software and that both the above copyright notice(s) and this + permission notice appear in supporting documentation. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. + IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE + BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, + OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, + WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, + ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS + SOFTWARE. + + Except as contained in this notice, the name of a copyright holder shall + not be used in advertising or otherwise to promote the sale, use or other + dealings in this Software without prior written authorization of the + copyright holder. + + +Parsing functionality provided by the NetCDF Java Library (http://www.unidata.ucar.edu/software/netcdf-java/) + + Copyright 1993-2010 University Corporation for Atmospheric Research/Unidata + + Portions of this software were developed by the Unidata Program at the University + Corporation for Atmospheric Research. + + Access and use of this software shall impose the following obligations and understandings + on the user. The user is granted the right, without any fee or cost, to use, copy, modify, + alter, enhance and distribute this software, and any derivative works thereof, and its + supporting documentation for any purpose whatsoever, provided that this entire notice + appears in all copies of the software, derivative works and supporting documentation. Further, + UCAR requests that the user credit UCAR/Unidata in any publications that result from the use + of this software or in any product that includes this software, although this is not an obligation. + The names UCAR and/or Unidata, however, may not be used in any advertising or publicity to endorse + or promote any products or commercial entity unless specific written permission is obtained from + UCAR/Unidata. The user also understands that UCAR/Unidata is not obligated to provide the user with + any support, consulting, training or assistance of any kind with regard to the use, operation and + performance of this software nor to provide the user with any updates, revisions, new versions or + "bug fixes." + + THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, + BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE ACCESS, + USE OR PERFORMANCE OF THIS SOFTWARE. + + +IPTC Photo Metadata descriptions are taken from the IPTC Photo Metadata +Standard, July 2010, Copyright 2010 International Press Telecommunications +Council. + + 1. The Specifications and Materials are licensed for use only on the condition that you agree to be bound by the terms of this license. Subject to this and other licensing requirements contained herein, you may, on a non-exclusive basis, use the Specifications and Materials. + 2. The IPTC openly provides the Specifications and Materials for voluntary use by individuals, partnerships, companies, corporations, organizations and any other entity for use at the entity's own risk. This disclaimer, license and release is intended to apply to the IPTC, its officers, directors, agents, representatives, members, contributors, affiliates, contractors, or co-venturers acting jointly or severally. + 3. The Document and translations thereof may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the copyright and license notices and references to the IPTC appearing in the Document and the terms of this Specifications License Agreement are included on all such copies and derivative works. Further, upon the receipt of written permission from the IPTC, the Document may be modified for the purpose of developing applications that use IPTC Specifications or as required to translate the Document into languages other than English. + 4. Any use, duplication, distribution, or exploitation of the Document and Specifications and Materials in any manner is at your own risk. + 5. NO WARRANTY, EXPRESSED OR IMPLIED, IS MADE REGARDING THE ACCURACY, ADEQUACY, COMPLETENESS, LEGALITY, RELIABILITY OR USEFULNESS OF ANY INFORMATION CONTAINED IN THE DOCUMENT OR IN ANY SPECIFICATION OR OTHER PRODUCT OR SERVICE PRODUCED OR SPONSORED BY THE IPTC. THE DOCUMENT AND THE INFORMATION CONTAINED HEREIN AND INCLUDED IN ANY SPECIFICATION OR OTHER PRODUCT OR SERVICE OF THE IPTC IS PROVIDED ON AN "AS IS" BASIS. THE IPTC DISCLAIMS ALL WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, ANY ACTUAL OR ASSERTED WARRANTY OF NON-INFRINGEMENT OF PROPRIETARY RIGHTS, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. NEITHER THE IPTC NOR ITS CONTRIBUTORS SHALL BE HELD LIABLE FOR ANY IMPROPER OR INCORRECT USE OF INFORMATION. NEITHER THE IPTC NOR ITS CONTRIBUTORS ASSUME ANY RESPONSIBILITY FOR ANYONE'S USE OF INFORMATION PROVIDED BY THE IPTC. IN NO EVENT SHALL THE IPTC OR ITS CONTRIBUTORS BE LIABLE TO ANYONE FOR DAMAGES OF ANY KIND, INCLUDING BUT NOT LIMITED TO, COMPENSATORY DAMAGES, LOST PROFITS, LOST DATA OR ANY FORM OF SPECIAL, INCIDENTAL, INDIRECT, CONSEQUENTIAL OR PUNITIVE DAMAGES OF ANY KIND WHETHER BASED ON BREACH OF CONTRACT OR WARRANTY, TORT, PRODUCT LIABILITY OR OTHERWISE. + 6. The IPTC takes no position regarding the validity or scope of any Intellectual Property or other rights that might be claimed to pertain to the implementation or use of the technology described in the Document or the extent to which any license under such rights might or might not be available. The IPTC does not represent that it has made any effort to identify any such rights. Copies of claims of rights made available for publication, assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of the Specifications and Materials, can be obtained from the Managing Director of the IPTC. + 7. By using the Specifications and Materials including the Document in any manner or for any purpose, you release the IPTC from all liabilities, claims, causes of action, allegations, losses, injuries, damages, or detriments of any nature arising from or relating to the use of the Specifications, Materials or any portion thereof. You further agree not to file a lawsuit, make a claim, or take any other formal or informal legal action against the IPTC, resulting from your acquisition, use, duplication, distribution, or exploitation of the Specifications, Materials or any portion thereof. Finally, you hereby agree that the IPTC is not liable for any direct, indirect, special or consequential damages arising from or relating to your acquisition, use, duplication, distribution, or exploitation of the Specifications, Materials or any portion thereof. + 8. Specifications and Materials may be downloaded or copied provided that ALL copies retain the ownership, copyright and license notices. + 9. Materials may not be edited, modified, or presented in a context that creates a misleading or false impression or statement as to the positions, actions, or statements of the IPTC. + 10. The name and trademarks of the IPTC may not be used in advertising, publicity, or in relation to products or services and their names without the specific, written prior permission of the IPTC. Any permitted use of the trademarks of the IPTC, whether registered or not, shall be accompanied by an appropriate mark and attribution, as agreed with the IPTC. + 11. Specifications may be extended by both members and non-members to provide additional functionality (Extension Specifications) provided that there is a clear recognition of the IPTC IP and its ownership in the Extension Specifications and the related documentation and provided that the extensions are clearly identified and provided that a perpetual license is granted by the creator of the Extension Specifications for other members and non-members to use the Extension Specifications and to continue extensions of the Extension Specifications. The IPTC does not waive any of its rights in the Specifications and Materials in this context. The Extension Specifications may be considered the intellectual property of their creator. The IPTC expressly disclaims any responsibility for damage caused by an extension to the Specifications. + 12. Specifications and Materials may be included in derivative work of both members and non-members provided that there is a clear recognition of the IPTC IP and its ownership in the derivative work and its related documentation. The IPTC does not waive any of its rights in the Specifications and Materials in this context. Derivative work in its entirety may be considered the intellectual property of the creator of the work .The IPTC expressly disclaims any responsibility for damage caused when its IP is used in a derivative context. + 13. This Specifications License Agreement is perpetual subject to your conformance to the terms of this Agreement. The IPTC may terminate this Specifications License Agreement immediately upon your breach of this Agreement and, upon such termination you will cease all use, duplication, distribution, and/or exploitation in any manner of the Specifications and Materials. + 14. This Specifications License Agreement reflects the entire agreement of the parties regarding the subject matter hereof and supersedes all prior agreements or representations regarding such matters, whether written or oral. To the extent any portion or provision of this Specifications License Agreement is found to be illegal or unenforceable, then the remaining provisions of this Specifications License Agreement will remain in full force and effect and the illegal or unenforceable provision will be construed to give it such effect as it may properly have that is consistent with the intentions of the parties. + 15. This Specifications License Agreement may only be modified in writing signed by an authorized representative of the IPTC. + 16. This Specifications License Agreement is governed by the law of United Kingdom, as such law is applied to contracts made and fully performed in the United Kingdom. Any disputes arising from or relating to this Specifications License Agreement will be resolved in the courts of the United Kingdom. You consent to the jurisdiction of such courts over you and covenant not to assert before such courts any objection to proceeding in such forums. + + +JUnRAR (https://github.com/edmund-wagner/junrar/) + + JUnRAR is based on the UnRAR tool, and covered by the same license + It was formerly available from http://java-unrar.svn.sourceforge.net/ + + ****** ***** ****** UnRAR - free utility for RAR archives + ** ** ** ** ** ** ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ****** ******* ****** License for use and distribution of + ** ** ** ** ** ** ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ** ** ** ** ** ** FREE portable version + ~~~~~~~~~~~~~~~~~~~~~ + + The source code of UnRAR utility is freeware. This means: + + 1. All copyrights to RAR and the utility UnRAR are exclusively + owned by the author - Alexander Roshal. + + 2. The UnRAR sources may be used in any software to handle RAR + archives without limitations free of charge, but cannot be used + to re-create the RAR compression algorithm, which is proprietary. + Distribution of modified UnRAR sources in separate form or as a + part of other software is permitted, provided that it is clearly + stated in the documentation and source comments that the code may + not be used to develop a RAR (WinRAR) compatible archiver. + + 3. The UnRAR utility may be freely distributed. It is allowed + to distribute UnRAR inside of other software packages. + + 4. THE RAR ARCHIVER AND THE UnRAR UTILITY ARE DISTRIBUTED "AS IS". + NO WARRANTY OF ANY KIND IS EXPRESSED OR IMPLIED. YOU USE AT + YOUR OWN RISK. THE AUTHOR WILL NOT BE LIABLE FOR DATA LOSS, + DAMAGES, LOSS OF PROFITS OR ANY OTHER KIND OF LOSS WHILE USING + OR MISUSING THIS SOFTWARE. + + 5. Installing and using the UnRAR utility signifies acceptance of + these terms and conditions of the license. + + 6. If you don't agree with terms of the license you must remove + UnRAR files from your storage devices and cease to use the + utility. + + Thank you for your interest in RAR and UnRAR. Alexander L. Roshal + +Sqlite (bundled in org.xerial's sqlite-jdbc) + This product bundles Sqlite, which is in the Public Domain. For details + see: https://www.sqlite.org/copyright.html diff --git a/plugins/mapper-attachments/licenses/tika-parsers-NOTICE.txt b/plugins/mapper-attachments/licenses/tika-parsers-NOTICE.txt new file mode 100644 index 00000000000..8e94f644b81 --- /dev/null +++ b/plugins/mapper-attachments/licenses/tika-parsers-NOTICE.txt @@ -0,0 +1,17 @@ +Apache Tika +Copyright 2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Copyright 1993-2010 University Corporation for Atmospheric Research/Unidata +This software contains code derived from UCAR/Unidata's NetCDF library. + +Tika-server component uses CDDL-licensed dependencies: jersey (http://jersey.java.net/) and +Grizzly (http://grizzly.java.net/) + +Tika-parsers component uses CDDL/LGPL dual-licensed dependency: jhighlight (https://github.com/codelibs/jhighlight) + +OpenCSV: Copyright 2005 Bytecode Pty Ltd. Licensed under the Apache License, Version 2.0 + +IPTC Photo Metadata descriptions Copyright 2010 International Press Telecommunications Council. diff --git a/plugins/mapper-attachments/licenses/xmlbeans-2.6.0.jar.sha1 b/plugins/mapper-attachments/licenses/xmlbeans-2.6.0.jar.sha1 new file mode 100644 index 00000000000..d27c56f66cb --- /dev/null +++ b/plugins/mapper-attachments/licenses/xmlbeans-2.6.0.jar.sha1 @@ -0,0 +1 @@ +29e80d2dd51f9dcdef8f9ffaee0d4dc1c9bbfc87 diff --git a/plugins/mapper-attachments/licenses/xmlbeans-LICENSE.txt b/plugins/mapper-attachments/licenses/xmlbeans-LICENSE.txt new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/plugins/mapper-attachments/licenses/xmlbeans-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/mapper-attachments/licenses/xmlbeans-NOTICE.txt b/plugins/mapper-attachments/licenses/xmlbeans-NOTICE.txt new file mode 100644 index 00000000000..906cc4c9684 --- /dev/null +++ b/plugins/mapper-attachments/licenses/xmlbeans-NOTICE.txt @@ -0,0 +1,29 @@ + ========================================================================= + == NOTICE file corresponding to section 4(d) of the Apache License, == + == Version 2.0, in this case for the Apache XmlBeans distribution. == + ========================================================================= + + This product includes software developed by + The Apache Software Foundation (http://www.apache.org/). + + Portions of this software were originally based on the following: + - software copyright (c) 2000-2003, BEA Systems, . + + Aside from contributions to the Apache XMLBeans project, this + software also includes: + + - one or more source files from the Apache Xerces-J and Apache Axis + products, Copyright (c) 1999-2003 Apache Software Foundation + + - W3C XML Schema documents Copyright 2001-2003 (c) World Wide Web + Consortium (Massachusetts Institute of Technology, European Research + Consortium for Informatics and Mathematics, Keio University) + + - resolver.jar from Apache Xml Commons project, + Copyright (c) 2001-2003 Apache Software Foundation + + - Piccolo XML Parser for Java from http://piccolo.sourceforge.net/, + Copyright 2002 Yuval Oren under the terms of the Apache Software License 2.0 + + - JSR-173 Streaming API for XML from http://sourceforge.net/projects/xmlpullparser/, + Copyright 2005 BEA under the terms of the Apache Software License 2.0 diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java new file mode 100644 index 00000000000..a52f0768082 --- /dev/null +++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java @@ -0,0 +1,655 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexOptions; +import org.apache.tika.language.LanguageIdentifier; +import org.apache.tika.metadata.Metadata; +import org.elasticsearch.Version; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.*; + +import java.io.IOException; +import java.util.*; + +import static org.elasticsearch.index.mapper.MapperBuilders.*; +import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField; +import static org.elasticsearch.index.mapper.core.TypeParsers.parsePathType; + +/** + * 

+ *      "field1" : "..."
+ *
+ *

Or: + * 

+ * {
+ *      "file1" : {
+ *          "_content_type" : "application/pdf",
+ *          "_content_length" : "500000000",
+ *          "_name" : "..../something.pdf",
+ *          "_content" : ""
+ *      }
+ * }
+ *
+ *

+ * _content_length = Specify the maximum amount of characters to extract from the attachment. If not specified, then the default for + * tika is 100,000 characters. Caution is required when setting large values as this can cause memory issues. + */ +public class AttachmentMapper extends FieldMapper { + + private static ESLogger logger = ESLoggerFactory.getLogger("mapper.attachment"); + + public static final String CONTENT_TYPE = "attachment"; + + public static class Defaults { + public static final ContentPath.Type PATH_TYPE = ContentPath.Type.FULL; + + public static final AttachmentFieldType FIELD_TYPE = new AttachmentFieldType(); + static { + FIELD_TYPE.freeze(); + } + } + + public static class FieldNames { + public static final String CONTENT = "content"; + public static final String TITLE = "title"; + public static final String NAME = "name"; + public static final String AUTHOR = "author"; + public static final String KEYWORDS = "keywords"; + public static final String DATE = "date"; + public static final String CONTENT_TYPE = "content_type"; + public static final String CONTENT_LENGTH = "content_length"; + public static final String LANGUAGE = "language"; + } + + static final class AttachmentFieldType extends MappedFieldType { + public AttachmentFieldType() {} + + protected AttachmentFieldType(AttachmentMapper.AttachmentFieldType ref) { + super(ref); + } + + public AttachmentMapper.AttachmentFieldType clone() { + return new AttachmentMapper.AttachmentFieldType(this); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + public String value(Object value) { + return value == null?null:value.toString(); + } + } + + public static class Builder extends FieldMapper.Builder { + + private ContentPath.Type pathType = Defaults.PATH_TYPE; + + private Boolean ignoreErrors = null; + + private Integer defaultIndexedChars = null; + + private Boolean langDetect = null; + + private Mapper.Builder contentBuilder; + + private Mapper.Builder titleBuilder = stringField(FieldNames.TITLE); + + private Mapper.Builder nameBuilder = stringField(FieldNames.NAME); + + private Mapper.Builder authorBuilder = stringField(FieldNames.AUTHOR); + + private Mapper.Builder keywordsBuilder = stringField(FieldNames.KEYWORDS); + + private Mapper.Builder dateBuilder = dateField(FieldNames.DATE); + + private Mapper.Builder contentTypeBuilder = stringField(FieldNames.CONTENT_TYPE); + + private Mapper.Builder contentLengthBuilder = integerField(FieldNames.CONTENT_LENGTH); + + private Mapper.Builder languageBuilder = stringField(FieldNames.LANGUAGE); + + public Builder(String name) { + super(name, new AttachmentFieldType()); + this.builder = this; + this.contentBuilder = stringField(FieldNames.CONTENT); + } + + public Builder pathType(ContentPath.Type pathType) { + this.pathType = pathType; + return this; + } + + public Builder content(Mapper.Builder content) { + this.contentBuilder = content; + return this; + } + + public Builder date(Mapper.Builder date) { + this.dateBuilder = date; + return this; + } + + public Builder author(Mapper.Builder author) { + this.authorBuilder = author; + return this; + } + + public Builder title(Mapper.Builder title) { + this.titleBuilder = title; + return this; + } + + public Builder name(Mapper.Builder name) { + this.nameBuilder = name; + return this; + } + + public Builder keywords(Mapper.Builder keywords) { + this.keywordsBuilder = keywords; + return this; + } + + public Builder contentType(Mapper.Builder contentType) { + this.contentTypeBuilder = contentType; + return this; + } + + public Builder contentLength(Mapper.Builder contentType) { + this.contentLengthBuilder = contentType; + return this; + } + + public Builder language(Mapper.Builder language) { + this.languageBuilder = language; + return this; + } + + @Override + public AttachmentMapper build(BuilderContext context) { + ContentPath.Type origPathType = context.path().pathType(); + context.path().pathType(pathType); + + FieldMapper contentMapper; + if (context.indexCreatedVersion().before(Version.V_2_0_0_beta1)) { + // old behavior, we need the content to be indexed under the attachment field name + if (contentBuilder instanceof FieldMapper.Builder == false) { + throw new IllegalStateException("content field for attachment must be a field mapper"); + } + ((FieldMapper.Builder)contentBuilder).indexName(name); + contentBuilder.name = name + "." + FieldNames.CONTENT; + contentMapper = (FieldMapper) contentBuilder.build(context); + context.path().add(name); + } else { + context.path().add(name); + contentMapper = (FieldMapper) contentBuilder.build(context); + } + + FieldMapper dateMapper = (FieldMapper) dateBuilder.build(context); + FieldMapper authorMapper = (FieldMapper) authorBuilder.build(context); + FieldMapper titleMapper = (FieldMapper) titleBuilder.build(context); + FieldMapper nameMapper = (FieldMapper) nameBuilder.build(context); + FieldMapper keywordsMapper = (FieldMapper) keywordsBuilder.build(context); + FieldMapper contentTypeMapper = (FieldMapper) contentTypeBuilder.build(context); + FieldMapper contentLength = (FieldMapper) contentLengthBuilder.build(context); + FieldMapper language = (FieldMapper) languageBuilder.build(context); + context.path().remove(); + + context.path().pathType(origPathType); + + if (defaultIndexedChars == null && context.indexSettings() != null) { + defaultIndexedChars = context.indexSettings().getAsInt("index.mapping.attachment.indexed_chars", 100000); + } + if (defaultIndexedChars == null) { + defaultIndexedChars = 100000; + } + + if (ignoreErrors == null && context.indexSettings() != null) { + ignoreErrors = context.indexSettings().getAsBoolean("index.mapping.attachment.ignore_errors", Boolean.TRUE); + } + if (ignoreErrors == null) { + ignoreErrors = Boolean.TRUE; + } + + if (langDetect == null && context.indexSettings() != null) { + langDetect = context.indexSettings().getAsBoolean("index.mapping.attachment.detect_language", Boolean.FALSE); + } + if (langDetect == null) { + langDetect = Boolean.FALSE; + } + MappedFieldType defaultFieldType = Defaults.FIELD_TYPE.clone(); + if(this.fieldType.indexOptions() != IndexOptions.NONE && !this.fieldType.tokenized()) { + defaultFieldType.setOmitNorms(true); + defaultFieldType.setIndexOptions(IndexOptions.DOCS); + if(!this.omitNormsSet && this.fieldType.boost() == 1.0F) { + this.fieldType.setOmitNorms(true); + } + + if(!this.indexOptionsSet) { + this.fieldType.setIndexOptions(IndexOptions.DOCS); + } + } + + defaultFieldType.freeze(); + this.setupFieldType(context); + return new AttachmentMapper(name, fieldType, defaultFieldType, pathType, defaultIndexedChars, ignoreErrors, langDetect, contentMapper, + dateMapper, titleMapper, nameMapper, authorMapper, keywordsMapper, contentTypeMapper, contentLength, + language, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); + } + } + + /** + * 

+     *  field1 : { type : "attachment" }
+     *
+ * Or: + * 
+     *  field1 : {
+     *      type : "attachment",
+     *      fields : {
+     *          content : {type : "binary"},
+     *          title : {store : "yes"},
+     *          date : {store : "yes"},
+     *          name : {store : "yes"},
+     *          author : {store : "yes"},
+     *          keywords : {store : "yes"},
+     *          content_type : {store : "yes"},
+     *          content_length : {store : "yes"}
+     *      }
+     * }
+     *
+ */ + public static class TypeParser implements Mapper.TypeParser { + + private Mapper.Builder findMapperBuilder(Map propNode, String propName, ParserContext parserContext) { + String type; + Object typeNode = propNode.get("type"); + if (typeNode != null) { + type = typeNode.toString(); + } else { + type = "string"; + } + Mapper.TypeParser typeParser = parserContext.typeParser(type); + Mapper.Builder mapperBuilder = typeParser.parse(propName, (Map) propNode, parserContext); + + return mapperBuilder; + } + + @SuppressWarnings({"unchecked"}) + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + AttachmentMapper.Builder builder = new AttachmentMapper.Builder(name); + + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String fieldName = entry.getKey(); + Object fieldNode = entry.getValue(); + if (fieldName.equals("path") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { + builder.pathType(parsePathType(name, fieldNode.toString())); + iterator.remove(); + } else if (fieldName.equals("fields")) { + Map fieldsNode = (Map) fieldNode; + for (Iterator> fieldsIterator = fieldsNode.entrySet().iterator(); fieldsIterator.hasNext();) { + Map.Entry entry1 = fieldsIterator.next(); + String propName = entry1.getKey(); + Map propNode = (Map) entry1.getValue(); + + Mapper.Builder mapperBuilder = findMapperBuilder(propNode, propName, parserContext); + if (parseMultiField((FieldMapper.Builder) mapperBuilder, fieldName, parserContext, propName, propNode)) { + fieldsIterator.remove(); + } else if (propName.equals(name) && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { + builder.content(mapperBuilder); + fieldsIterator.remove(); + } else { + switch (propName) { + case FieldNames.CONTENT: + builder.content(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.DATE: + builder.date(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.AUTHOR: + builder.author(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.CONTENT_LENGTH: + builder.contentLength(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.CONTENT_TYPE: + builder.contentType(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.KEYWORDS: + builder.keywords(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.LANGUAGE: + builder.language(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.TITLE: + builder.title(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.NAME: + builder.name(mapperBuilder); + fieldsIterator.remove(); + break; + } + } + } + DocumentMapperParser.checkNoRemainingFields(fieldName, fieldsNode, parserContext.indexVersionCreated()); + iterator.remove(); + } + } + + return builder; + } + } + + private final ContentPath.Type pathType; + + private final int defaultIndexedChars; + + private final boolean ignoreErrors; + + private final boolean defaultLangDetect; + + private final FieldMapper contentMapper; + + private final FieldMapper dateMapper; + + private final FieldMapper authorMapper; + + private final FieldMapper titleMapper; + + private final FieldMapper nameMapper; + + private final FieldMapper keywordsMapper; + + private final FieldMapper contentTypeMapper; + + private final FieldMapper contentLengthMapper; + + private final FieldMapper languageMapper; + + public AttachmentMapper(String simpleName, MappedFieldType type, MappedFieldType defaultFieldType, ContentPath.Type pathType, int defaultIndexedChars, Boolean ignoreErrors, + Boolean defaultLangDetect, FieldMapper contentMapper, + FieldMapper dateMapper, FieldMapper titleMapper, FieldMapper nameMapper, FieldMapper authorMapper, + FieldMapper keywordsMapper, FieldMapper contentTypeMapper, FieldMapper contentLengthMapper, + FieldMapper languageMapper, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { + super(simpleName, type, defaultFieldType, indexSettings, multiFields, copyTo); + this.pathType = pathType; + this.defaultIndexedChars = defaultIndexedChars; + this.ignoreErrors = ignoreErrors; + this.defaultLangDetect = defaultLangDetect; + this.contentMapper = contentMapper; + this.dateMapper = dateMapper; + this.titleMapper = titleMapper; + this.nameMapper = nameMapper; + this.authorMapper = authorMapper; + this.keywordsMapper = keywordsMapper; + this.contentTypeMapper = contentTypeMapper; + this.contentLengthMapper = contentLengthMapper; + this.languageMapper = languageMapper; + } + + @Override + public Mapper parse(ParseContext context) throws IOException { + byte[] content = null; + String contentType = null; + int indexedChars = defaultIndexedChars; + boolean langDetect = defaultLangDetect; + String name = null; + String language = null; + + XContentParser parser = context.parser(); + XContentParser.Token token = parser.currentToken(); + if (token == XContentParser.Token.VALUE_STRING) { + content = parser.binaryValue(); + } else { + String currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.VALUE_STRING) { + if ("_content".equals(currentFieldName)) { + content = parser.binaryValue(); + } else if ("_content_type".equals(currentFieldName)) { + contentType = parser.text(); + } else if ("_name".equals(currentFieldName)) { + name = parser.text(); + } else if ("_language".equals(currentFieldName)) { + language = parser.text(); + } + } else if (token == XContentParser.Token.VALUE_NUMBER) { + if ("_indexed_chars".equals(currentFieldName) || "_indexedChars".equals(currentFieldName)) { + indexedChars = parser.intValue(); + } + } else if (token == XContentParser.Token.VALUE_BOOLEAN) { + if ("_detect_language".equals(currentFieldName) || "_detectLanguage".equals(currentFieldName)) { + langDetect = parser.booleanValue(); + } + } + } + } + + // Throw clean exception when no content is provided Fix #23 + if (content == null) { + throw new MapperParsingException("No content is provided."); + } + + Metadata metadata = new Metadata(); + if (contentType != null) { + metadata.add(Metadata.CONTENT_TYPE, contentType); + } + if (name != null) { + metadata.add(Metadata.RESOURCE_NAME_KEY, name); + } + + String parsedContent; + try { + parsedContent = TikaImpl.parse(content, metadata, indexedChars); + } catch (Throwable e) { + // #18: we could ignore errors when Tika does not parse data + if (!ignoreErrors) { + logger.trace("exception caught", e); + throw new MapperParsingException("Failed to extract [" + indexedChars + "] characters of text for [" + name + "] : " + + e.getMessage(), e); + } else { + logger.debug("Failed to extract [{}] characters of text for [{}]: [{}]", indexedChars, name, e.getMessage()); + logger.trace("exception caught", e); + } + return null; + } + + context = context.createExternalValueContext(parsedContent); + contentMapper.parse(context); + + if (langDetect) { + try { + if (language != null) { + metadata.add(Metadata.CONTENT_LANGUAGE, language); + } else { + LanguageIdentifier identifier = new LanguageIdentifier(parsedContent); + language = identifier.getLanguage(); + } + context = context.createExternalValueContext(language); + languageMapper.parse(context); + } catch(Throwable t) { + logger.debug("Cannot detect language: [{}]", t.getMessage()); + } + } + + if (name != null) { + try { + context = context.createExternalValueContext(name); + nameMapper.parse(context); + } catch(MapperParsingException e){ + if (!ignoreErrors) throw e; + if (logger.isDebugEnabled()) logger.debug("Ignoring MapperParsingException catch while parsing name: [{}]", + e.getMessage()); + } + } + + if (metadata.get(Metadata.DATE) != null) { + try { + context = context.createExternalValueContext(metadata.get(Metadata.DATE)); + dateMapper.parse(context); + } catch(MapperParsingException e){ + if (!ignoreErrors) throw e; + if (logger.isDebugEnabled()) logger.debug("Ignoring MapperParsingException catch while parsing date: [{}]: [{}]", + e.getMessage(), context.externalValue()); + } + } + + if (metadata.get(Metadata.TITLE) != null) { + try { + context = context.createExternalValueContext(metadata.get(Metadata.TITLE)); + titleMapper.parse(context); + } catch(MapperParsingException e){ + if (!ignoreErrors) throw e; + if (logger.isDebugEnabled()) logger.debug("Ignoring MapperParsingException catch while parsing title: [{}]: [{}]", + e.getMessage(), context.externalValue()); + } + } + + if (metadata.get(Metadata.AUTHOR) != null) { + try { + context = context.createExternalValueContext(metadata.get(Metadata.AUTHOR)); + authorMapper.parse(context); + } catch(MapperParsingException e){ + if (!ignoreErrors) throw e; + if (logger.isDebugEnabled()) logger.debug("Ignoring MapperParsingException catch while parsing author: [{}]: [{}]", + e.getMessage(), context.externalValue()); + } + } + + if (metadata.get(Metadata.KEYWORDS) != null) { + try { + context = context.createExternalValueContext(metadata.get(Metadata.KEYWORDS)); + keywordsMapper.parse(context); + } catch(MapperParsingException e){ + if (!ignoreErrors) throw e; + if (logger.isDebugEnabled()) logger.debug("Ignoring MapperParsingException catch while parsing keywords: [{}]: [{}]", + e.getMessage(), context.externalValue()); + } + } + + if (contentType == null) { + contentType = metadata.get(Metadata.CONTENT_TYPE); + } + if (contentType != null) { + try { + context = context.createExternalValueContext(contentType); + contentTypeMapper.parse(context); + } catch(MapperParsingException e){ + if (!ignoreErrors) throw e; + if (logger.isDebugEnabled()) logger.debug("Ignoring MapperParsingException catch while parsing content_type: [{}]: [{}]", e.getMessage(), context.externalValue()); + } + } + + int length = content.length; + // If we have CONTENT_LENGTH from Tika we use it + if (metadata.get(Metadata.CONTENT_LENGTH) != null) { + length = Integer.parseInt(metadata.get(Metadata.CONTENT_LENGTH)); + } + + try { + context = context.createExternalValueContext(length); + contentLengthMapper.parse(context); + } catch(MapperParsingException e){ + if (!ignoreErrors) throw e; + if (logger.isDebugEnabled()) logger.debug("Ignoring MapperParsingException catch while parsing content_length: [{}]: [{}]", e.getMessage(), context.externalValue()); + } + +// multiFields.parse(this, context); + + return null; + } + + @Override + protected void parseCreateField(ParseContext parseContext, List fields) throws IOException { + + } + + @Override + public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + // ignore this for now + } + + @Override + @SuppressWarnings("unchecked") + public Iterator iterator() { + List extras = Arrays.asList( + contentMapper, + dateMapper, + titleMapper, + nameMapper, + authorMapper, + keywordsMapper, + contentTypeMapper, + contentLengthMapper, + languageMapper); + return Iterators.concat(super.iterator(), extras.iterator()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(name()); + builder.field("type", CONTENT_TYPE); + if (indexCreatedBefore2x) { + builder.field("path", pathType.name().toLowerCase(Locale.ROOT)); + } + + builder.startObject("fields"); + contentMapper.toXContent(builder, params); + authorMapper.toXContent(builder, params); + titleMapper.toXContent(builder, params); + nameMapper.toXContent(builder, params); + dateMapper.toXContent(builder, params); + keywordsMapper.toXContent(builder, params); + contentTypeMapper.toXContent(builder, params); + contentLengthMapper.toXContent(builder, params); + languageMapper.toXContent(builder, params); + multiFields.toXContent(builder, params); + builder.endObject(); + + multiFields.toXContent(builder, params); + builder.endObject(); + return builder; + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } +} diff --git a/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java similarity index 55% rename from plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy rename to plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java index 66810451688..e38b02b7b55 100644 --- a/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java @@ -17,7 +17,25 @@ * under the License. */ -grant { - // needed because of problems in aws-sdk - permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; -}; +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.index.IndexService; +import org.elasticsearch.plugins.Plugin; + +public class MapperAttachmentsPlugin extends Plugin { + + @Override + public String name() { + return "mapper-attachments"; + } + + @Override + public String description() { + return "Adds the attachment type allowing to parse difference attachment formats"; + } + + @Override + public void onIndexService(IndexService indexService) { + indexService.mapperService().documentMapperParser().putTypeParser("attachment", new AttachmentMapper.TypeParser()); + } +} diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/TikaImpl.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/TikaImpl.java new file mode 100644 index 00000000000..38e292725a5 --- /dev/null +++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/TikaImpl.java @@ -0,0 +1,159 @@ +package org.elasticsearch.mapper.attachments; + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.ByteArrayInputStream; +import java.io.FilePermission; +import java.io.IOException; +import java.lang.reflect.ReflectPermission; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Path; +import java.security.AccessControlContext; +import java.security.AccessController; +import java.security.PermissionCollection; +import java.security.Permissions; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.security.ProtectionDomain; +import java.security.SecurityPermission; +import java.util.PropertyPermission; + +import org.apache.tika.Tika; +import org.apache.tika.exception.TikaException; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.AutoDetectParser; +import org.apache.tika.parser.Parser; +import org.elasticsearch.SpecialPermission; +import org.elasticsearch.bootstrap.JarHell; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.io.PathUtils; + +/** + * Runs tika with limited parsers and limited permissions. + *

+ * Do NOT make public + */ +final class TikaImpl { + + /** subset of parsers for types we support */ + private static final Parser PARSERS[] = new Parser[] { + // documents + new org.apache.tika.parser.html.HtmlParser(), + new org.apache.tika.parser.rtf.RTFParser(), + new org.apache.tika.parser.pdf.PDFParser(), + new org.apache.tika.parser.txt.TXTParser(), + new org.apache.tika.parser.microsoft.OfficeParser(), + new org.apache.tika.parser.microsoft.OldExcelParser(), + new org.apache.tika.parser.microsoft.ooxml.OOXMLParser(), + new org.apache.tika.parser.odf.OpenDocumentParser(), + new org.apache.tika.parser.iwork.IWorkPackageParser(), + new org.apache.tika.parser.xml.DcXMLParser(), + new org.apache.tika.parser.epub.EpubParser(), + }; + + /** autodetector based on this subset */ + private static final AutoDetectParser PARSER_INSTANCE = new AutoDetectParser(PARSERS); + + /** singleton tika instance */ + private static final Tika TIKA_INSTANCE = new Tika(PARSER_INSTANCE.getDetector(), PARSER_INSTANCE); + + /** + * parses with tika, throwing any exception hit while parsing the document + */ + // only package private for testing! + static String parse(final byte content[], final Metadata metadata, final int limit) throws TikaException, IOException { + // check that its not unprivileged code like a script + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + + try { + return AccessController.doPrivileged(new PrivilegedExceptionAction() { + @Override + public String run() throws TikaException, IOException { + return TIKA_INSTANCE.parseToString(new ByteArrayInputStream(content), metadata, limit); + } + }, RESTRICTED_CONTEXT); + } catch (PrivilegedActionException e) { + // checked exception from tika: unbox it + Throwable cause = e.getCause(); + if (cause instanceof TikaException) { + throw (TikaException) cause; + } else if (cause instanceof IOException) { + throw (IOException) cause; + } else { + throw new AssertionError(cause); + } + } + } + + // apply additional containment for parsers, this is intersected with the current permissions + // its hairy, but worth it so we don't have some XML flaw reading random crap from the FS + private static final AccessControlContext RESTRICTED_CONTEXT = new AccessControlContext( + new ProtectionDomain[] { + new ProtectionDomain(null, getRestrictedPermissions()) + } + ); + + // compute some minimal permissions for parsers. they only get r/w access to the java temp directory, + // the ability to load some resources from JARs, and read sysprops + static PermissionCollection getRestrictedPermissions() { + Permissions perms = new Permissions(); + // property/env access needed for parsing + perms.add(new PropertyPermission("*", "read")); + perms.add(new RuntimePermission("getenv.TIKA_CONFIG")); + + // add permissions for resource access: + // classpath + addReadPermissions(perms, JarHell.parseClassPath()); + // plugin jars + if (TikaImpl.class.getClassLoader() instanceof URLClassLoader) { + addReadPermissions(perms, ((URLClassLoader)TikaImpl.class.getClassLoader()).getURLs()); + } + // jvm's java.io.tmpdir (needs read/write) + perms.add(new FilePermission(System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + "-", + "read,readlink,write,delete")); + // current hacks needed for POI/PDFbox issues: + perms.add(new SecurityPermission("putProviderProperty.BC")); + perms.add(new SecurityPermission("insertProvider")); + perms.add(new ReflectPermission("suppressAccessChecks")); + perms.setReadOnly(); + return perms; + } + + // add resources to (what is typically) a jar, but might not be (e.g. in tests/IDE) + @SuppressForbidden(reason = "adds access to jar resources") + static void addReadPermissions(Permissions perms, URL resources[]) { + try { + for (URL url : resources) { + Path path = PathUtils.get(url.toURI()); + // resource itself + perms.add(new FilePermission(path.toString(), "read,readlink")); + // classes underneath + perms.add(new FilePermission(path.toString() + System.getProperty("file.separator") + "-", "read,readlink")); + } + } catch (URISyntaxException bogus) { + throw new RuntimeException(bogus); + } + } +} diff --git a/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy b/plugins/mapper-attachments/src/main/plugin-metadata/plugin-security.policy similarity index 63% rename from plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy rename to plugins/mapper-attachments/src/main/plugin-metadata/plugin-security.policy index 66810451688..e23e9f4d0cf 100644 --- a/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/mapper-attachments/src/main/plugin-metadata/plugin-security.policy @@ -17,7 +17,14 @@ * under the License. */ +// NOTE: when modifying this file, look at restrictions in TikaImpl too grant { - // needed because of problems in aws-sdk + // needed to apply additional sandboxing to tika parsing + permission java.security.SecurityPermission "createAccessControlContext"; + + // TODO: fix PDFBox not to actually install bouncy castle like this + permission java.security.SecurityPermission "putProviderProperty.BC"; + permission java.security.SecurityPermission "insertProvider"; + // TODO: fix POI XWPF to not do this: https://bz.apache.org/bugzilla/show_bug.cgi?id=58597 permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java new file mode 100644 index 00000000000..9378f2d71f9 --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +public class AttachmentUnitTestCase extends ESTestCase { + + protected Settings testSettings; + + @Before + public void createSettings() throws Exception { + testSettings = Settings.builder() + .put("path.home", createTempDir()) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id) + .build(); + } +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java new file mode 100644 index 00000000000..4833dfb4f34 --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java @@ -0,0 +1,52 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.core.StringFieldMapper; +import org.elasticsearch.mapper.attachments.AttachmentMapper; +import org.junit.Before; + +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.instanceOf; + +/** + * + */ +public class DateAttachmentMapperTests extends AttachmentUnitTestCase { + + private DocumentMapperParser mapperParser; + + @Before + public void setupMapperParser() throws Exception { + mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); + mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + } + + public void testSimpleMappings() throws Exception { + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/date/date-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + + // Our mapping should be kept as a String + assertThat(docMapper.mappers().getMapper("file.date"), instanceOf(StringFieldMapper.class)); + } +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java new file mode 100644 index 00000000000..28b40dbb895 --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java @@ -0,0 +1,135 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.mapper.attachments.AttachmentMapper; + +import java.io.IOException; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.*; + +/** + * Test for https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/18 + * Note that we have converted /org/elasticsearch/index/mapper/xcontent/testContentLength.txt + * to a /org/elasticsearch/index/mapper/xcontent/encrypted.pdf with password `12345678`. + */ +public class EncryptedDocMapperTests extends AttachmentUnitTestCase { + + public void testMultipleDocsEncryptedLast() throws IOException { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); + mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html"); + byte[] pdf = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/encrypted.pdf"); + + BytesReference json = jsonBuilder() + .startObject() + .field("file1", html) + .field("file2", pdf) + .endObject().bytes(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + assertThat(doc.get(docMapper.mappers().getMapper("file1.content").fieldType().names().indexName()), containsString("World")); + assertThat(doc.get(docMapper.mappers().getMapper("file1.title").fieldType().names().indexName()), equalTo("Hello")); + assertThat(doc.get(docMapper.mappers().getMapper("file1.author").fieldType().names().indexName()), equalTo("kimchy")); + assertThat(doc.get(docMapper.mappers().getMapper("file1.keywords").fieldType().names().indexName()), equalTo("elasticsearch,cool,bonsai")); + assertThat(doc.get(docMapper.mappers().getMapper("file1.content_type").fieldType().names().indexName()), equalTo("text/html; charset=ISO-8859-1")); + assertThat(doc.getField(docMapper.mappers().getMapper("file1.content_length").fieldType().names().indexName()).numericValue().longValue(), is(344L)); + + assertThat(doc.get(docMapper.mappers().getMapper("file2").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file2.title").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file2.author").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file2.keywords").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file2.content_type").fieldType().names().indexName()), nullValue()); + assertThat(doc.getField(docMapper.mappers().getMapper("file2.content_length").fieldType().names().indexName()), nullValue()); + } + + public void testMultipleDocsEncryptedFirst() throws IOException { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); + mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html"); + byte[] pdf = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/encrypted.pdf"); + + BytesReference json = jsonBuilder() + .startObject() + .field("file1", pdf) + .field("file2", html) + .endObject().bytes(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + assertThat(doc.get(docMapper.mappers().getMapper("file1").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file1.title").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file1.author").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file1.keywords").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file1.content_type").fieldType().names().indexName()), nullValue()); + assertThat(doc.getField(docMapper.mappers().getMapper("file1.content_length").fieldType().names().indexName()), nullValue()); + + assertThat(doc.get(docMapper.mappers().getMapper("file2.content").fieldType().names().indexName()), containsString("World")); + assertThat(doc.get(docMapper.mappers().getMapper("file2.title").fieldType().names().indexName()), equalTo("Hello")); + assertThat(doc.get(docMapper.mappers().getMapper("file2.author").fieldType().names().indexName()), equalTo("kimchy")); + assertThat(doc.get(docMapper.mappers().getMapper("file2.keywords").fieldType().names().indexName()), equalTo("elasticsearch,cool,bonsai")); + assertThat(doc.get(docMapper.mappers().getMapper("file2.content_type").fieldType().names().indexName()), equalTo("text/html; charset=ISO-8859-1")); + assertThat(doc.getField(docMapper.mappers().getMapper("file2.content_length").fieldType().names().indexName()).numericValue().longValue(), is(344L)); + } + + public void testMultipleDocsEncryptedNotIgnoringErrors() throws IOException { + try { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), + Settings.builder() + .put("index.mapping.attachment.ignore_errors", false) + .build()).documentMapperParser(); + mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html"); + byte[] pdf = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/encrypted.pdf"); + + BytesReference json = jsonBuilder() + .startObject() + .field("file1", pdf) + .field("file2", html) + .endObject().bytes(); + + docMapper.parse("person", "person", "1", json); + fail("Expected doc parsing exception"); + } catch (MapperParsingException e) { + if (e.getMessage() == null || e.getMessage().contains("is encrypted") == false) { + // wrong exception + throw e; + } + } + } + +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java new file mode 100644 index 00000000000..4a5a1d1f8ab --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java @@ -0,0 +1,128 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.core.StringFieldMapper; +import org.elasticsearch.mapper.attachments.AttachmentMapper; +import org.junit.Before; + +import java.io.IOException; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +/** + * + */ +public class LanguageDetectionAttachmentMapperTests extends AttachmentUnitTestCase { + + private DocumentMapper docMapper; + + @Before + public void setupMapperParser() throws IOException { + setupMapperParser(true); + } + + public void setupMapperParser(boolean langDetect) throws IOException { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), + Settings.settingsBuilder() + .put("index.mapping.attachment.detect_language", langDetect) + .build()).documentMapperParser(); + mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/language/language-mapping.json"); + docMapper = mapperParser.parse(mapping); + + assertThat(docMapper.mappers().getMapper("file.language"), instanceOf(StringFieldMapper.class)); + } + + private void testLanguage(String filename, String expected, String... forcedLanguage) throws IOException { + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename); + + XContentBuilder xcb = jsonBuilder() + .startObject() + .startObject("file") + .field("_name", filename) + .field("_content", html); + + if (forcedLanguage.length > 0) { + xcb.field("_language", forcedLanguage[0]); + } + + xcb.endObject().endObject(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", xcb.bytes()).rootDoc(); + + // Our mapping should be kept as a String + assertThat(doc.get(docMapper.mappers().getMapper("file.language").fieldType().names().indexName()), equalTo(expected)); + } + + public void testFrDetection() throws Exception { + testLanguage("text-in-french.txt", "fr"); + } + + public void testEnDetection() throws Exception { + testLanguage("text-in-english.txt", "en"); + } + + public void testFrForced() throws Exception { + testLanguage("text-in-english.txt", "fr", "fr"); + } + + /** + * This test gives strange results! detection of ":-)" gives "lt" as a result + */ + public void testNoLanguage() throws Exception { + testLanguage("text-in-nolang.txt", "lt"); + } + + public void testLangDetectDisabled() throws Exception { + // We replace the mapper with another one which have index.mapping.attachment.detect_language = false + setupMapperParser(false); + testLanguage("text-in-english.txt", null); + } + + public void testLangDetectDocumentEnabled() throws Exception { + // We replace the mapper with another one which have index.mapping.attachment.detect_language = false + setupMapperParser(false); + + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-english.txt"); + + XContentBuilder xcb = jsonBuilder() + .startObject() + .startObject("file") + .field("_name", "text-in-english.txt") + .field("_content", html) + .field("_detect_language", true) + .endObject().endObject(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", xcb.bytes()).rootDoc(); + + // Our mapping should be kept as a String + assertThat(doc.get(docMapper.mappers().getMapper("file.language").fieldType().names().indexName()), equalTo("en")); + } +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperAttachmentsRestIT.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperAttachmentsRestIT.java new file mode 100644 index 00000000000..bdbafea710a --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperAttachmentsRestIT.java @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.mapper.attachments.MapperAttachmentsPlugin; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.RestTestCandidate; +import org.elasticsearch.test.rest.parser.RestTestParseException; + +import java.io.IOException; + +public class MapperAttachmentsRestIT extends ESRestTestCase { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("plugin.types", MapperAttachmentsPlugin.class.getName()) + .build(); + } + + public MapperAttachmentsRestIT(@Name("yaml") RestTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException, RestTestParseException { + return createParameters(0, 1); + } +} + diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperTestUtils.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperTestUtils.java new file mode 100644 index 00000000000..deb29f12a42 --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperTestUtils.java @@ -0,0 +1,53 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.analysis.AnalysisService; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.test.IndexSettingsModule; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collections; + +class MapperTestUtils { + + public static MapperService newMapperService(Path tempDir, Settings indexSettings) throws IOException { + Settings nodeSettings = Settings.builder() + .put("path.home", tempDir) + .build(); + indexSettings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(indexSettings) + .build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("test"), indexSettings, Collections.emptyList()); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(nodeSettings)).build(idxSettings); + SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap()); + return new MapperService(idxSettings, analysisService, similarityService); + } +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java new file mode 100644 index 00000000000..80543ff4feb --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java @@ -0,0 +1,104 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.mapper.attachments.AttachmentMapper; + +import java.io.IOException; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.*; + +/** + * Test for https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/38 + */ +public class MetadataMapperTests extends AttachmentUnitTestCase { + + protected void checkMeta(String filename, Settings otherSettings, Long expectedDate, Long expectedLength) throws IOException { + Settings settings = Settings.builder() + .put(this.testSettings) + .put(otherSettings) + .build(); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), settings).documentMapperParser(); + mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/metadata/test-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename); + + BytesReference json = jsonBuilder() + .startObject() + .startObject("file") + .field("_name", filename) + .field("_content", html) + .endObject() + .endObject().bytes(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + assertThat(doc.get(docMapper.mappers().getMapper("file.content").fieldType().names().indexName()), containsString("World")); + assertThat(doc.get(docMapper.mappers().getMapper("file.name").fieldType().names().indexName()), equalTo(filename)); + if (expectedDate == null) { + assertThat(doc.getField(docMapper.mappers().getMapper("file.date").fieldType().names().indexName()), nullValue()); + } else { + assertThat(doc.getField(docMapper.mappers().getMapper("file.date").fieldType().names().indexName()).numericValue().longValue(), is(expectedDate)); + } + assertThat(doc.get(docMapper.mappers().getMapper("file.title").fieldType().names().indexName()), equalTo("Hello")); + assertThat(doc.get(docMapper.mappers().getMapper("file.author").fieldType().names().indexName()), equalTo("kimchy")); + assertThat(doc.get(docMapper.mappers().getMapper("file.keywords").fieldType().names().indexName()), equalTo("elasticsearch,cool,bonsai")); + assertThat(doc.get(docMapper.mappers().getMapper("file.content_type").fieldType().names().indexName()), equalTo("text/html; charset=ISO-8859-1")); + assertThat(doc.getField(docMapper.mappers().getMapper("file.content_length").fieldType().names().indexName()).numericValue().longValue(), is(expectedLength)); + } + + public void testIgnoreWithoutDate() throws Exception { + checkMeta("htmlWithoutDateMeta.html", Settings.builder().build(), null, 300L); + } + + public void testIgnoreWithEmptyDate() throws Exception { + checkMeta("htmlWithEmptyDateMeta.html", Settings.builder().build(), null, 334L); + } + + public void testIgnoreWithCorrectDate() throws Exception { + checkMeta("htmlWithValidDateMeta.html", Settings.builder().build(), 1354233600000L, 344L); + } + + public void testWithoutDate() throws Exception { + checkMeta("htmlWithoutDateMeta.html", Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(), null, 300L); + } + + public void testWithEmptyDate() throws Exception { + try { + checkMeta("htmlWithEmptyDateMeta.html", Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(), null, null); + } catch (MapperParsingException expected) { + assertTrue(expected.getMessage().contains("failed to parse")); + } + } + + public void testWithCorrectDate() throws Exception { + checkMeta("htmlWithValidDateMeta.html", Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(), 1354233600000L, 344L); + } +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java new file mode 100644 index 00000000000..d7535df22f8 --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java @@ -0,0 +1,154 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.common.Base64; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.core.DateFieldMapper; +import org.elasticsearch.index.mapper.core.StringFieldMapper; +import org.elasticsearch.mapper.attachments.AttachmentMapper; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.nio.charset.StandardCharsets; + +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.*; + +/** + * + */ +public class MultifieldAttachmentMapperTests extends AttachmentUnitTestCase { + + private DocumentMapperParser mapperParser; + private ThreadPool threadPool; + + @Before + public void setupMapperParser() throws Exception { + mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); + mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + + } + + @After + public void cleanup() throws InterruptedException { + terminate(threadPool); + } + + public void testSimpleMappings() throws Exception { + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + + + assertThat(docMapper.mappers().getMapper("file.content"), instanceOf(StringFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("file.content.suggest"), instanceOf(StringFieldMapper.class)); + + assertThat(docMapper.mappers().getMapper("file.date"), instanceOf(DateFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("file.date.string"), instanceOf(StringFieldMapper.class)); + + assertThat(docMapper.mappers().getMapper("file.title"), instanceOf(StringFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("file.title.suggest"), instanceOf(StringFieldMapper.class)); + + assertThat(docMapper.mappers().getMapper("file.name"), instanceOf(StringFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("file.name.suggest"), instanceOf(StringFieldMapper.class)); + + assertThat(docMapper.mappers().getMapper("file.author"), instanceOf(StringFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("file.author.suggest"), instanceOf(StringFieldMapper.class)); + + assertThat(docMapper.mappers().getMapper("file.keywords"), instanceOf(StringFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("file.keywords.suggest"), instanceOf(StringFieldMapper.class)); + + assertThat(docMapper.mappers().getMapper("file.content_type"), instanceOf(StringFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("file.content_type.suggest"), instanceOf(StringFieldMapper.class)); + } + + public void testExternalValues() throws Exception { + String originalText = "This is an elasticsearch mapper attachment test."; + String contentType = "text/plain; charset=ISO-8859-1"; + String forcedName = "dummyname.txt"; + + String bytes = Base64.encodeBytes(originalText.getBytes(StandardCharsets.ISO_8859_1)); + threadPool = new ThreadPool("testing-only"); + + MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY); + mapperService.documentMapperParser().putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json"); + + DocumentMapper documentMapper = mapperService.documentMapperParser().parse(mapping); + + ParsedDocument doc = documentMapper.parse("person", "person", "1", XContentFactory.jsonBuilder() + .startObject() + .field("file", bytes) + .endObject() + .bytes()); + + assertThat(doc.rootDoc().getField("file.content"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content").stringValue(), is(originalText + "\n")); + + assertThat(doc.rootDoc().getField("file.content_type"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content_type").stringValue(), is(contentType)); + assertThat(doc.rootDoc().getField("file.content_type.suggest"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content_type.suggest").stringValue(), is(contentType)); + assertThat(doc.rootDoc().getField("file.content_length"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content_length").numericValue().intValue(), is(originalText.length())); + + assertThat(doc.rootDoc().getField("file.content.suggest"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content.suggest").stringValue(), is(originalText + "\n")); + + // Let's force some values + doc = documentMapper.parse("person", "person", "1", XContentFactory.jsonBuilder() + .startObject() + .startObject("file") + .field("_content", bytes) + .field("_name", forcedName) + .endObject() + .endObject() + .bytes()); + + assertThat(doc.rootDoc().getField("file.content"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content").stringValue(), is(originalText + "\n")); + + assertThat(doc.rootDoc().getField("file.content_type"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content_type").stringValue(), is(contentType)); + assertThat(doc.rootDoc().getField("file.content_type.suggest"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content_type.suggest").stringValue(), is(contentType)); + assertThat(doc.rootDoc().getField("file.content_length"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content_length").numericValue().intValue(), is(originalText.length())); + + assertThat(doc.rootDoc().getField("file.content.suggest"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content.suggest").stringValue(), is(originalText + "\n")); + + assertThat(doc.rootDoc().getField("file.name"), notNullValue()); + assertThat(doc.rootDoc().getField("file.name").stringValue(), is(forcedName)); + // In mapping we have default store:false + assertThat(doc.rootDoc().getField("file.name").fieldType().stored(), is(false)); + assertThat(doc.rootDoc().getField("file.name.suggest"), notNullValue()); + assertThat(doc.rootDoc().getField("file.name.suggest").stringValue(), is(forcedName)); + // In mapping we set store:true for suggest subfield + assertThat(doc.rootDoc().getField("file.name.suggest").fieldType().stored(), is(true)); + } +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java new file mode 100644 index 00000000000..651d063beaa --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java @@ -0,0 +1,114 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.mapper.attachments.AttachmentMapper; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.*; + +/** + * + */ +public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase { + + public void testSimpleMappings() throws Exception { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); + mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html"); + + BytesReference json = jsonBuilder().startObject().field("file", html).endObject().bytes(); + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + + assertThat(doc.get(docMapper.mappers().getMapper("file.content_type").fieldType().names().indexName()), startsWith("application/xhtml+xml")); + assertThat(doc.get(docMapper.mappers().getMapper("file.title").fieldType().names().indexName()), equalTo("XHTML test document")); + assertThat(doc.get(docMapper.mappers().getMapper("file.content").fieldType().names().indexName()), containsString("This document tests the ability of Apache Tika to extract content")); + + // re-parse it + String builtMapping = docMapper.mappingSource().string(); + docMapper = mapperParser.parse(builtMapping); + + json = jsonBuilder().startObject().field("file", html).endObject().bytes(); + + doc = docMapper.parse("person", "person", "1", json).rootDoc(); + + assertThat(doc.get(docMapper.mappers().getMapper("file.content_type").fieldType().names().indexName()), startsWith("application/xhtml+xml")); + assertThat(doc.get(docMapper.mappers().getMapper("file.title").fieldType().names().indexName()), equalTo("XHTML test document")); + assertThat(doc.get(docMapper.mappers().getMapper("file.content").fieldType().names().indexName()), containsString("This document tests the ability of Apache Tika to extract content")); + } + + public void testContentBackcompat() throws Exception { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), + Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id) + .build()).documentMapperParser(); + mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html"); + + BytesReference json = jsonBuilder().startObject().field("file", html).endObject().bytes(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + assertThat(doc.get("file"), containsString("This document tests the ability of Apache Tika to extract content")); + } + + /** + * test for https://github.com/elastic/elasticsearch-mapper-attachments/issues/179 + */ + public void testSimpleMappingsWithAllFields() throws Exception { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); + mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping-all-fields.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html"); + + BytesReference json = jsonBuilder().startObject().field("file", html).endObject().bytes(); + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + + assertThat(doc.get(docMapper.mappers().getMapper("file.content_type").fieldType().names().indexName()), startsWith("application/xhtml+xml")); + assertThat(doc.get(docMapper.mappers().getMapper("file.title").fieldType().names().indexName()), equalTo("XHTML test document")); + assertThat(doc.get(docMapper.mappers().getMapper("file.content").fieldType().names().indexName()), containsString("This document tests the ability of Apache Tika to extract content")); + + // re-parse it + String builtMapping = docMapper.mappingSource().string(); + docMapper = mapperParser.parse(builtMapping); + + json = jsonBuilder().startObject().field("file", html).endObject().bytes(); + + doc = docMapper.parse("person", "person", "1", json).rootDoc(); + + assertThat(doc.get(docMapper.mappers().getMapper("file.content_type").fieldType().names().indexName()), startsWith("application/xhtml+xml")); + assertThat(doc.get(docMapper.mappers().getMapper("file.title").fieldType().names().indexName()), equalTo("XHTML test document")); + assertThat(doc.get(docMapper.mappers().getMapper("file.content").fieldType().names().indexName()), containsString("This document tests the ability of Apache Tika to extract content")); + } + +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java new file mode 100644 index 00000000000..80de4d91d15 --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java @@ -0,0 +1,191 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.apache.commons.cli.CommandLine; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.cli.CliTool; +import org.elasticsearch.common.cli.CliToolConfig; +import org.elasticsearch.common.cli.Terminal; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.mapper.attachments.AttachmentMapper; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Locale; + +import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd; +import static org.elasticsearch.common.cli.CliToolConfig.Builder.option; +import static org.elasticsearch.common.io.Streams.copy; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; + +/** + * This class provides a simple main class which can be used to test what is extracted from a given binary file. + * You can run it using + * -u file://URL/TO/YOUR/DOC + * --size set extracted size (default to mapper attachment size) + * BASE64 encoded binary + * + * Example: + * StandaloneRunner BASE64Text + * StandaloneRunner -u /tmp/mydoc.pdf + * StandaloneRunner -u /tmp/mydoc.pdf --size 1000000 + */ +public class StandaloneRunner extends CliTool { + + private static final CliToolConfig CONFIG = CliToolConfig.config("tika", StandaloneRunner.class) + .cmds(TikaRunner.CMD) + .build(); + + static { + System.setProperty("es.path.home", "/tmp"); + } + + static class TikaRunner extends Command { + private static final String NAME = "tika"; + private final String url; + private final Integer size; + private final String base64text; + private final DocumentMapper docMapper; + + private static final CliToolConfig.Cmd CMD = cmd(NAME, TikaRunner.class) + .options(option("u", "url").required(false).hasArg(false)) + .options(option("t", "size").required(false).hasArg(false)) + .build(); + + protected TikaRunner(Terminal terminal, String url, Integer size, String base64text) throws IOException { + super(terminal); + this.size = size; + this.url = url; + this.base64text = base64text; + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(PathUtils.get("."), Settings.EMPTY).documentMapperParser(); // use CWD b/c it won't be used + mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/standalone/standalone-mapping.json"); + docMapper = mapperParser.parse(mapping); + } + + @Override + public ExitStatus execute(Settings settings, Environment env) throws Exception { + XContentBuilder builder = jsonBuilder().startObject().field("file").startObject(); + + if (base64text != null) { + // If base64 is provided + builder.field("_content", base64text); + } else { + // A file is provided + byte[] bytes = copyToBytes(PathUtils.get(url)); + builder.field("_content", bytes); + } + + if (size >= 0) { + builder.field("_indexed_chars", size); + } + + BytesReference json = builder.endObject().endObject().bytes(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + + terminal.println("## Extracted text"); + terminal.println("--------------------- BEGIN -----------------------"); + terminal.println("%s", doc.get("file.content")); + terminal.println("---------------------- END ------------------------"); + terminal.println("## Metadata"); + printMetadataContent(doc, AttachmentMapper.FieldNames.AUTHOR); + printMetadataContent(doc, AttachmentMapper.FieldNames.CONTENT_LENGTH); + printMetadataContent(doc, AttachmentMapper.FieldNames.CONTENT_TYPE); + printMetadataContent(doc, AttachmentMapper.FieldNames.DATE); + printMetadataContent(doc, AttachmentMapper.FieldNames.KEYWORDS); + printMetadataContent(doc, AttachmentMapper.FieldNames.LANGUAGE); + printMetadataContent(doc, AttachmentMapper.FieldNames.NAME); + printMetadataContent(doc, AttachmentMapper.FieldNames.TITLE); + + return ExitStatus.OK; + } + + private void printMetadataContent(ParseContext.Document doc, String field) { + terminal.println("- %s: %s", field, doc.get(docMapper.mappers().getMapper("file." + field).fieldType().names().indexName())); + } + + public static byte[] copyToBytes(Path path) throws IOException { + try (InputStream is = Files.newInputStream(path)) { + if (is == null) { + throw new FileNotFoundException("Resource [" + path + "] not found in classpath"); + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + copy(is, out); + return out.bytes().toBytes(); + } + } + } + + public static Command parse(Terminal terminal, CommandLine cli) throws IOException { + String url = cli.getOptionValue("u"); + String base64text = null; + String sSize = cli.getOptionValue("size"); + Integer size = sSize != null ? Integer.parseInt(sSize) : -1; + if (url == null && cli.getArgs().length == 0) { + return exitCmd(ExitStatus.USAGE, terminal, "url or BASE64 content should be provided (type -h for help)"); + } + if (url == null) { + if (cli.getArgs().length == 0) { + return exitCmd(ExitStatus.USAGE, terminal, "url or BASE64 content should be provided (type -h for help)"); + } + base64text = cli.getArgs()[0]; + } else { + if (cli.getArgs().length == 1) { + return exitCmd(ExitStatus.USAGE, terminal, "url or BASE64 content should be provided. Not both. (type -h for help)"); + } + } + return new TikaRunner(terminal, url, size, base64text); + } + } + + public StandaloneRunner() { + super(CONFIG); + } + + + public static void main(String[] args) { + StandaloneRunner pluginManager = new StandaloneRunner(); + pluginManager.execute(args); + } + + @Override + protected Command parse(String cmdName, CommandLine cli) throws Exception { + switch (cmdName.toLowerCase(Locale.ROOT)) { + case TikaRunner.NAME: return TikaRunner.parse(terminal, cli); + default: + assert false : "can't get here as cmd name is validated before this method is called"; + return exitCmd(ExitStatus.CODE_ERROR); + } + } +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java new file mode 100644 index 00000000000..a5e3ec9c17c --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java @@ -0,0 +1,66 @@ +package org.elasticsearch.mapper.attachments; + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; +import org.apache.lucene.util.TestUtil; +import org.apache.tika.metadata.Metadata; + +import org.elasticsearch.test.ESTestCase; + +/** + * Evil test-coverage cheat, we parse a bunch of docs from tika + * so that we have a nice grab-bag variety, and assert some content + * comes back and no exception. + */ +@SuppressFileSystems("ExtrasFS") // don't try to parse extraN +public class TikaDocTests extends ESTestCase { + + /** some test files from tika test suite, zipped up */ + static final String TIKA_FILES = "/org/elasticsearch/index/mapper/attachment/test/tika-files.zip"; + + public void testFiles() throws Exception { + Path tmp = createTempDir(); + TestUtil.unzip(getClass().getResourceAsStream(TIKA_FILES), tmp); + + try (DirectoryStream stream = Files.newDirectoryStream(tmp)) { + for (Path doc : stream) { + logger.debug("parsing: {}", doc); + assertParseable(doc); + } + } + } + + void assertParseable(Path fileName) throws Exception { + try { + byte bytes[] = Files.readAllBytes(fileName); + String parsedContent = TikaImpl.parse(bytes, new Metadata(), -1); + assertNotNull(parsedContent); + assertFalse(parsedContent.isEmpty()); + logger.debug("extracted content: {}", parsedContent); + } catch (Throwable e) { + throw new RuntimeException("parsing of filename: " + fileName.getFileName() + " failed", e); + } + } +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaImplTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaImplTests.java new file mode 100644 index 00000000000..fc17d59603f --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaImplTests.java @@ -0,0 +1,11 @@ +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.test.ESTestCase; + +public class TikaImplTests extends ESTestCase { + + public void testTikaLoads() throws Exception { + Class.forName("org.elasticsearch.mapper.attachments.TikaImpl"); + } + +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java new file mode 100644 index 00000000000..88c15c10c54 --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java @@ -0,0 +1,171 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.apache.tika.io.IOUtils; +import org.apache.tika.metadata.Metadata; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.mapper.attachments.AttachmentMapper; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.mapper.attachments.AttachmentMapper.FieldNames.*; +import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.isEmptyOrNullString; +import static org.hamcrest.Matchers.not; + +/** + * Test for different documents + */ +public class VariousDocTests extends AttachmentUnitTestCase { + + protected DocumentMapper docMapper; + + @Before + public void createMapper() throws IOException { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); + mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/various-doc/test-mapping.json"); + docMapper = mapperParser.parse(mapping); + } + + /** + * Test for https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/104 + */ + public void testWordDocxDocument104() throws Exception { + assertParseable("issue-104.docx"); + testMapper("issue-104.docx", false); + } + + /** + * Test for encrypted PDF + */ + public void testEncryptedPDFDocument() throws Exception { + assertException("encrypted.pdf", "is encrypted"); + // TODO Remove when this will be fixed in Tika. See https://issues.apache.org/jira/browse/TIKA-1548 + System.clearProperty("sun.font.fontmanager"); + testMapper("encrypted.pdf", true); + } + + /** + * Test for HTML + */ + public void testHtmlDocument() throws Exception { + assertParseable("htmlWithEmptyDateMeta.html"); + testMapper("htmlWithEmptyDateMeta.html", false); + } + + /** + * Test for XHTML + */ + public void testXHtmlDocument() throws Exception { + assertParseable("testXHTML.html"); + testMapper("testXHTML.html", false); + } + + /** + * Test for TXT + */ + public void testTxtDocument() throws Exception { + assertParseable("text-in-english.txt"); + testMapper("text-in-english.txt", false); + } + + /** + * Test for .epub + */ + public void testEpubDocument() throws Exception { + assertParseable("testEPUB.epub"); + testMapper("testEPUB.epub", false); + } + + /** + * Test for ASCIIDOC + * Not yet supported by Tika: https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/29 + */ + public void testAsciidocDocument() throws Exception { + assertParseable("asciidoc.asciidoc"); + testMapper("asciidoc.asciidoc", false); + } + + void assertException(String filename, String expectedMessage) throws Exception { + try (InputStream is = VariousDocTests.class.getResourceAsStream("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename)) { + byte bytes[] = IOUtils.toByteArray(is); + TikaImpl.parse(bytes, new Metadata(), -1); + fail("expected exception"); + } catch (Exception e) { + if (e.getMessage() != null && e.getMessage().contains(expectedMessage)) { + // ok + } else { + // unexpected + throw e; + } + } + } + + protected void assertParseable(String filename) throws Exception { + try (InputStream is = VariousDocTests.class.getResourceAsStream("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename)) { + byte bytes[] = IOUtils.toByteArray(is); + String parsedContent = TikaImpl.parse(bytes, new Metadata(), -1); + assertThat(parsedContent, not(isEmptyOrNullString())); + logger.debug("extracted content: {}", parsedContent); + } + } + + protected void testMapper(String filename, boolean errorExpected) throws IOException { + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename); + + BytesReference json = jsonBuilder() + .startObject() + .startObject("file") + .field("_name", filename) + .field("_content", html) + .endObject() + .endObject().bytes(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + if (!errorExpected) { + assertThat(doc.get(docMapper.mappers().getMapper("file.content").fieldType().names().indexName()), not(isEmptyOrNullString())); + logger.debug("-> extracted content: {}", doc.get(docMapper.mappers().getMapper("file").fieldType().names().indexName())); + logger.debug("-> extracted metadata:"); + printMetadataContent(doc, AUTHOR); + printMetadataContent(doc, CONTENT_LENGTH); + printMetadataContent(doc, CONTENT_TYPE); + printMetadataContent(doc, DATE); + printMetadataContent(doc, KEYWORDS); + printMetadataContent(doc, LANGUAGE); + printMetadataContent(doc, NAME); + printMetadataContent(doc, TITLE); + } + } + + private void printMetadataContent(ParseContext.Document doc, String field) { + logger.debug("- [{}]: [{}]", field, doc.get(docMapper.mappers().getMapper("file." + field).fieldType().names().indexName())); + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/asciidoc.asciidoc b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/asciidoc.asciidoc new file mode 100644 index 00000000000..dc06d4e83dd --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/asciidoc.asciidoc @@ -0,0 +1,5 @@ +[[tika-asciidoc]] += AsciiDoc test + +Here is a test of the asciidoc format. + diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/encrypted.pdf b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/encrypted.pdf new file mode 100644 index 00000000000..569a904a315 Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/encrypted.pdf differ diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithEmptyDateMeta.html b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithEmptyDateMeta.html new file mode 100644 index 00000000000..f151208e384 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithEmptyDateMeta.html @@ -0,0 +1,11 @@ + + + + Hello + + + + +World + diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html new file mode 100644 index 00000000000..79b5a6234ec --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html @@ -0,0 +1,11 @@ + + + + Hello + + + + +World + diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithoutDateMeta.html b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithoutDateMeta.html new file mode 100644 index 00000000000..3322fa3a734 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithoutDateMeta.html @@ -0,0 +1,10 @@ + + + + Hello + + + +World + diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/issue-104.docx b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/issue-104.docx new file mode 100644 index 00000000000..f126e20b32e Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/issue-104.docx differ diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testContentLength.txt b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testContentLength.txt new file mode 100644 index 00000000000..d392c2d0979 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testContentLength.txt @@ -0,0 +1,9 @@ +Begin + +BeforeLimit AfterLimit + +Broadway + +Nearing the end + +End \ No newline at end of file diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testEPUB.epub b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testEPUB.epub new file mode 100644 index 00000000000..a6fc2e634d5 Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testEPUB.epub differ diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html new file mode 100644 index 00000000000..f5564f025d2 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html @@ -0,0 +1,29 @@ + + + + XHTML test document + + + + +

+ This document tests the ability of Apache Tika to extract content + from an XHTML document. +

+ + diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-english.txt b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-english.txt new file mode 100644 index 00000000000..08280926034 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-english.txt @@ -0,0 +1 @@ +"God Save the Queen" (alternatively "God Save the King" \ No newline at end of file diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-french.txt b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-french.txt new file mode 100644 index 00000000000..e4619fb1b88 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-french.txt @@ -0,0 +1 @@ +Allons enfants de la Patrie Le jour de gloire est arrivé. Contre nous de la tyrannie \ No newline at end of file diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-nolang.txt b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-nolang.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/standalone/standalone-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/standalone/standalone-mapping.json new file mode 100644 index 00000000000..c8680cf0644 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/standalone/standalone-mapping.json @@ -0,0 +1,9 @@ +{ + "person":{ + "properties":{ + "file":{ + "type":"attachment" + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files.zip new file mode 100644 index 00000000000..10f5d507677 Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files.zip differ diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/date/date-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/date/date-mapping.json new file mode 100644 index 00000000000..c4c90ce75e9 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/date/date-mapping.json @@ -0,0 +1,12 @@ +{ + "person": { + "properties": { + "file": { + "type": "attachment", + "fields": { + "date": { "type": "string" } + } + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json new file mode 100644 index 00000000000..7dc796c2b17 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json @@ -0,0 +1,12 @@ +{ + "person":{ + "properties":{ + "file1":{ + "type":"attachment" + }, + "file2":{ + "type":"attachment" + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/language/language-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/language/language-mapping.json new file mode 100644 index 00000000000..02176c7ca0d --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/language/language-mapping.json @@ -0,0 +1,12 @@ +{ + "person": { + "properties": { + "file": { + "type": "attachment", + "fields": { + "language": { "type": "string" } + } + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/metadata/test-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/metadata/test-mapping.json new file mode 100644 index 00000000000..c8680cf0644 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/metadata/test-mapping.json @@ -0,0 +1,9 @@ +{ + "person":{ + "properties":{ + "file":{ + "type":"attachment" + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json new file mode 100644 index 00000000000..314c70db2aa --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json @@ -0,0 +1,56 @@ +{ + "person": { + "properties": { + "file": { + "type": "attachment", + "fields": { + "content": { + "type": "string", + "fields": { + "suggest": { "type": "string" } + } + }, + "date": { + "type": "date", + "fields": { + "string": { "type": "string" } + } + }, + "title": { + "type": "string", + "fields": { + "suggest": { "type": "string" } + } + }, + "name": { + "type": "string", + "fields": { + "suggest": { + "type": "string", + "store": true + } + } + }, + "author": { + "type": "string", + "fields": { + "suggest": { "type": "string" } + } + }, + "keywords": { + "type": "string", + "fields": { + "suggest": { "type": "string" } + } + }, + "content_type": { + "type": "string", + "fields": { + "suggest": { "type": "string" } + } + } + } + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping-all-fields.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping-all-fields.json new file mode 100644 index 00000000000..ea83b98ceec --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping-all-fields.json @@ -0,0 +1,19 @@ +{ + "person":{ + "properties":{ + "file":{ + "type":"attachment", + "fields" : { + "content" : {"store" : "yes"}, + "title" : {"store" : "yes"}, + "date" : {"store" : "yes"}, + "author" : {"analyzer" : "standard"}, + "keywords" : {"store" : "yes"}, + "content_type" : {"store" : "yes"}, + "content_length" : {"store" : "yes"}, + "language" : {"store" : "yes"} + } + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json new file mode 100644 index 00000000000..c8680cf0644 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json @@ -0,0 +1,9 @@ +{ + "person":{ + "properties":{ + "file":{ + "type":"attachment" + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/various-doc/test-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/various-doc/test-mapping.json new file mode 100644 index 00000000000..c8680cf0644 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/various-doc/test-mapping.json @@ -0,0 +1,9 @@ +{ + "person":{ + "properties":{ + "file":{ + "type":"attachment" + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/00_basic.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/00_basic.yaml new file mode 100644 index 00000000000..819478d7d56 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/00_basic.yaml @@ -0,0 +1,14 @@ +# Integration tests for plugin: check name is correct +# +"Mapper attachments loaded": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - match: { nodes.$master.plugins.0.name: mapper-attachments } + - match: { nodes.$master.plugins.0.jvm: true } diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/10_index.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/10_index.yaml new file mode 100644 index 00000000000..8b2c4b0ea18 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/10_index.yaml @@ -0,0 +1,158 @@ +# Integration tests for Mapper Attachments plugin +# + +--- +# https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/23 +"Index empty attachment": + + - do: + indices.create: + index: test + body: + mappings: + doc: + properties: + file: + type: attachment + - do: + cluster.health: + wait_for_status: yellow + + - do: + catch: /(.)*mapper_parsing_exception.+No content is provided\.(.)*/ + index: + index: test + type: doc + id: 1 + body: + file: { } + +--- +# https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/18 +# Encoded content with https://www.base64encode.org/ +# File1 +# +# +# +# Hello +# +# +# +# +#World +# +# File2 is an encrypted PDF with a password + +"Multiple Attachments With Encrypted Doc Ignore Failures": + + - do: + indices.create: + index: test + body: + settings: + index.mapping.attachment.ignore_errors: true + mappings: + doc: + properties: + file1: + type: attachment + file2: + type: attachment + - do: + cluster.health: + wait_for_status: yellow + + - do: + index: + index: test + type: doc + id: 1 + body: + file1: "PCFET0NUWVBFIEhUTUwgUFVCTElDICItLy9XM0MvL0RURCBIVE1MIDQuMDEgVHJhbnNpdGlvbmFsLy9FTiINCiAgICAgICAgImh0dHA6Ly93d3cudzMub3JnL1RSL2h0bWw0L2xvb3NlLmR0ZCI+DQo8aHRtbCBsYW5nPSJmciI+DQo8aGVhZD4NCiAgICA8dGl0bGU+SGVsbG88L3RpdGxlPg0KICAgIDxtZXRhIG5hbWU9ImRhdGUiIGNvbnRlbnQ9IjIwMTItMTEtMzAiPg0KICAgIDxtZXRhIG5hbWU9IkF1dGhvciIgY29udGVudD0ia2ltY2h5Ij4NCiAgICA8bWV0YSBuYW1lPSJLZXl3b3JkcyIgY29udGVudD0iZWxhc3RpY3NlYXJjaCxjb29sLGJvbnNhaSI+DQo8L2hlYWQ+DQo8Ym9keT5Xb3JsZDwvYm9keT4NCjwvaHRtbD4NCg==" + file2: "JVBERi0xLjQNCiXDpMO8w7bDnw0KMiAwIG9iag0KPDwvTGVuZ3RoIDMgMCBSL0ZpbHRlci9GbGF0ZURlY29kZT4+DQpzdHJlYW0NCjB+az8/Pz8/bDU/Nyw+Pz/Ln1hqZT8uTD9LMS0/Jj9bRDM/SFY/Pz92P2B2P1dxRE5BP2k8Wk0/P01RP3c/PytUP2YmPz9OSD94WT8/RD8/P31AID9JZ9KeYT91Pz8/RD8/PURPP0g/P28/bz8/P3k/UVk/dD8/Pzg/P2tjP3lmPz8/Pz9uP01mfEV6dVwmP095P2osPz14Xyk/P0swbQ0KZW5kc3RyZWFtDQplbmRvYmoNCg0KMyAwIG9iag0KMTg2DQplbmRvYmoNCg0KNCAwIG9iag0KPDwvVHlwZS9YT2JqZWN0DQovU3VidHlwZS9Gb3JtDQovQkJveFsgLTkgNDIwIDYwNCA0MjAuMSBdDQovR3JvdXA8PC9TL1RyYW5zcGFyZW5jeS9DUy9EZXZpY2VSR0IvSyB0cnVlPj4NCi9MZW5ndGggOA0KL0ZpbHRlci9GbGF0ZURlY29kZQ0KPj4NCnN0cmVhbQ0KPz8gPw0K24kNCmVuZHN0cmVhbQ0KZW5kb2JqDQoNCjUgMCBvYmoNCjw8L0NBIDAuNQ0KICAgL2NhIDAuNQ0KPj4NCmVuZG9iag0KDQo3IDAgb2JqDQo8PC9MZW5ndGggOCAwIFIvRmlsdGVyL0ZsYXRlRGVjb2RlL0xlbmd0aDEgMTk5MjA+Pg0Kc3RyZWFtDQo/Rz/Lij9lP1E/UT8/MCs/M0I/N00/P00/DQpfdz87ND8/TD8/QT8/Nz8/wrU/Pz9zPzlPUyQ8XT/Vsj9DP9yOT1ljPz8/Uj8/ITY/Pz9Jaz8/dT8/Pw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB7Pz8/Pz8/P3k/dj8/ICl0Ij9sNz8/Pz8/bWs/P9uuPz9zVSM3Pz8zP1A/P3U/VT8sP1wpPz96P0o/PT8/ID8/OkA/Ii52w6YyP+WHmj8/Pz8/dV0/P2U/KD8/P08/P3FBP25Yc3w4P3U/QHs/Pyx5WD9OY9iEYcu3Pz8/P2JfSz8mPz8/Pz8/QTI/Kig/Pz95Pz8/P20/P04/P2w/DQo/P8mxWD8/Pz8/P1Yva3l0bEs/eVxrP3c/TDZRPz9DIT8/P1s/LD8/P2Y/Oj9mPyAgPy5vP2E/DQo/ej/Qsj8/Pz8/TDZRV2k/Pz8oPz86MT8/bXd4P3IkPz9SWT8/Kz9UIcOBLD8tPz8/Pz9LP2DbnT8/P3w9Pj8/P28/Pz9UYUY/fCxOPT8/P04/cj9kP0I/DQpUUD8/zZs/WT/Zk1YmPz8/XcmQYCI/44WwPz8tP2t8Mis/c1A6Pz9NP08/ez8/T1Y/1KY/P30/Pz8/Pz8/Uj8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzDQo/dz9qPz9QP013Pz9PP3E/fD/Rqz8NCiAgICAgICAgICAgICAgICAgICAgPw0KICAgICAgICAgICAgICAgICAgICAgfEE0P1RoPz8/dmsqPz9tPz8/DQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/Pz8/Pz86Kg0KP966Qk9bPzg/LD8/QT8/P1g/ICAgICAgID8/Wz8/Pz9aP3t5Q3I/fjt8TT8/Yz9Cw7gxRz8/Pz8/1qE/P34/P29DPz9GP3E4P2U/Pz9xPz9rWD9uP1Q/CT8/aduZ66aaPz8oCWc/Sj8/eD8/1IsNCj9MMD8/N3N4P3R+Pz8/JD8/Pz8/Kj8/P1Y/P0NWOj8+Pz9hPz9rP3E/cz8/P9yTWik/Pz/Gjz9zPz/Mvm06YNaXRc6tZT8/dNuePz89SzAoP2o/fT8/ej8/NEA/YD8/Nz9PW1k/LD8vVzA/P2I/PzFRP0I8Pz8NCj8/Rj8vPz8zP8WLPzE/Pz/hp6k/Pz/aq0trIF4/P1lUPz8/Iy5uWT8/RGc/P1g/Pz8/xJs/Pz8/IW0/Pz9GDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/SdyZP3fPsk8/xK1PP3FSP08zPz9cPzJ4Pz5rL0ZKYT8nU3VZCTR1fj81aj9hPz9p34XwuLOLSj/Ogz8/ez8gPz91ZT/Ej2o/P0k/Pz8/ST9lLj8/Tz/EhD9VRXs/OUE/P2N0P3pOXz8/cz8/Pz9oUD8/Pz8/V00/P1soPz8/Nz88NFo/Pz96Pyg/Pz82Pz/GqkI/QD84Uj9RP9uoMnk/LiFBPw0KPzQ/QSo3P3w/1YI/X3w/ID/atT/pm4g/Rj8/Nj9CPj8/PyQ/P2A/Pz9QQFw/Pz8/az/cgj8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/TD8/MD8/Pz96P1w/Pz8/Pz8/Vz9JUiR+Pz9nP3stbz/Sp1I/aHE/P1drP2ljPzQiPz9CND8kID9YPz9uPz9m7JCocj91Lj8/MiE/ND9IPz8/PzM/REg/Pz9dP9ioPyc/Ync/PyM/P0Y4Py0/04g/yKY/dz9XQz8/Pz8oej8NClw/Vc6gPz9xPy8/Pz8wDQogICAgICAgICAgICAgPzFjVz8oPyY1fT8/XFY/TjM/P18/Pz8/QCE/P34/PzZRUz8/Pz8/Pz8/R3RyMj8/CVo/Tj8/Mw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9cPyo/Uj8/Pz8oPz8/Pz8xPyU/Wlw/PzQ/dT87P3o/P1c/KM6PPz/XmT8/Pz85PD8zPz9+Pz8xDQo/P9Wlbz8/YduHLj90djwmPz8tdz9vaD/Trz86Pz8yPz8pJz8/MT8nPwl8PyNT07U6Pz8/NtyHPzI/QT8qPzQxTT8/aj8/Yj8/P2pBWlQ/PzZJNT8/PzrNvT8/ID8/P2M/P1A/DQogICAgICAgICAgICAgICA/Pz82L3vMo1NdLT8lPz8vPz8/P2MxOz9qIyQjwoFkPwk/Pz9kbT8/LTc/dj8/Pz95P35uP3RNP3A35oeRIXw/Mj8/ND8/NEY/Ij/KjDc/cD/Wj30uP1M/IygNCiAgICAgICAgICAgICAgICAgICAgICBOPz8/DQogICAgICAgICAgICAgICAgICAgICAgICAgID08Pz94MD8/IT9KZT8/P20/Pz/hjLE/SD9uP1A/Sig/cVR6CT8/Tz8/P0/TvDk/1aw/Pz8oZj9LP0dAPz8/ez8/IWlCP15fMnY/VFYyOD9IPy4/WD8/ez8/Pz9DP0Y8adGF36w3LD8/cj/Rhj8/X1M6bT8/SD8/X1I/M2gJP2c/bFU2P3c/Pz8/VT8/Pz8/dj8/Pz8/P1h5IUM/biQ/Pz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgID9yPz8/P0rprrsnPz8/KykqTDlTPz8/Pz8/Py8/Qz8/Pz8/WG1lPz8/P3g/P1w/NSUiW34nPz9Ecj8/JX0/SHVfR0k/QUFPP9WzPz8/az8/Lz96QD8/YU0/KD9SYj8/Pz8/PyclRQ0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBZPT8/xKo/Pz9qUnk/UFlLPz8/P3t3XX0/UD/Zgz8/Pz8oPyE/Pz8/MzE/Pw0KICAgICAgICAgdD8/Pz9UPz8/dj94PT8/Pz8/Pz9SPzM/Pz8tKj9uSHk/PzDNkz9yUT8/3ZVaP24/Pz9hP3c/aj/Qgz8gPz9RPy8hdT9mXWclP1g/dHs/PWlgP3o/Qz9tXT9TZD0/cD8lP2g/DQogICAgICAgICAgICAgICAgICAgICAgICA/XD8/Pz8/ZT8/P2LZqmt+Pz96N96KP3Q/Ijk/Pz8/KjM/P3k/U2g/PyBnPz8/SD9rP1k/Pz8/R1A0Pz96P3Q/YENRP+eAjz8/Xj9RP3U/P05KP2NbND9xP9GVP3k/Uj8/Jz9vcj8hPyU/P2BMPz9iYz9RPz8/P3E/cD9kP1Q/Oz8/Pz93Pz8/Pz9dQkouP8+kP3NtP1UNCj92PzlvPz89KHs/PT8/cD9manc/Pz8/Tz8/Pz8/Pz9ZPz8/dnA/TD9+Pz9c1os/Pz8/aX4/JD8/Pz8+Pz8/CT/Vr0k/dz8/a3w/JlYNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgP35SP3U/fCY/QmE/P1ZWPz8/Pz9HKz9FJD9rSj85Pj91d09IP0Q/Pz8/Pz9hPy04Pz8/Pz9wPzE/XkQ3P3N6P1s/K1JTWj0/Pzk/Tj9Qaj9HP3s/Pz9NNA0Kej8/P2h9Pyh6Pzg/OD8/Pz/ejC4/Jz8/Jz9RUj8/QD8/JD8/JFgNCj8/Pw0KICAgPz8/PD8/WEk/P2g/cz9wPD90Pz8/PzRhP3pIRS0/e1U/Pyk/P9SrVnRYJGY/Pz9qPyIvPz9tTH0/Zdy5I1I/Q8KfPz8/Pz9CMT8/ZD9lbz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB1Kz50PtSDdEs/UT8/Pz9tXD8/Pz9PyZExP2FOPj8vNHBuVQ0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA7Pj8qZj88P21aST8/Yj8xQD9oPz8hdT9eP3zMrj8sK21hPz8wMCc/ej9bPz8/XVw/YFpNUCI/aUE/Pz8/7IKRNGpFMz91P1RyPz9wP1BSPz8/Tj9lJH5nPz9wPz8/TkduPz8/U30/cT9SPz9Laz8/L34/P9q2P0kzPz/dpF86Pw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQj9cP04/dD5oaT8/Oz8/amssZz/LtD8/ej/FuiE/P3dmPz8/Pz9Ze3s/Pz8/Lj9wPm8oeG5626hFbX4zQj8/P28uPz8/Pzs/Pz8/CT97Mz8/Sz8iJi9VIj95Pz8/VT8/Pz8NCiAgICAgICAgICAgICAgPz9kTT8/Pz9VKz8/Lj9zP3dTzbtAPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/MSY/Pz9oP0g/Wz9KNz9mPz8/xZk/Mz94MjQ/OGR3Zz8/Jj8/PzB5Sj9DPz8lO8qNbj9vcj9NbT8Jw5BoPz8/TWEsPyE/Pz9sDQp5Kj94Pzc/aloiPz88Xz8/ej9APz8jPz9IPz0/1Lc/UD9XOVs/XD8/Pz8/P0huP3E9RD/Urz8/Pz8/NS1bPz8/bD8/P3tfZj8/PD90NdauPz/cvD8/P0o/XT9IfD8/Ij8kPz8/P24/Pz8nPz8/PzchLHZRPz/Mij8/PA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICo/XnFhP2o/XT8/RDzXlz8/xIQ/P1lGPz8/Rd6eITA/U1Y/P20/Pz9wXj8/MD9ePz83P9KIP2A/Pz81Zj8/Pz85TT8/Wz8/P24/ej8vPz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgMD8/PHJIPz8/P34/PwlTLTA0fHU/P1pMPz8sPz8/zpc3TC0/x6A/PyEyPz8/Pz8/Ij96xa8/Sj9ddz8/Pz8/P3ZjVT8/PyA/cD8/PzpY7ZWOOlc/bD8jQGo/Py4/Pz8/Pz9ePz8/Pz8/Pz9kTjNdRy1mT3t0Pz0/ajkgSDQ/VnI/Pz8/Pz0/P2gnJD98P0lHPz91PztzNT8/Py/KsD98Pz8/Nj9wPz8/UlY/Vz9nPyw/P1p7NT96cyFvPyB0Kz8pP1dFPw0KPz8/Kj8/J8meP2E/Pz8/Pz9AXyFjPz8/PyZkx45lIj9FPz84Pz9vZT8/Pz8/YD8/P28/Pz86PykyPz8rPz9xPz8/Py0/LUhwPz/csG4/P3ghPz8wP0J5P8qjZk8/dEo/Pz9OUj82P2JWLHbKujM/DQogICAgICAgICAgICAgICAgICAgICA/Lm5AYD9Wczs/P0I/P2k/Pz8oP0c/NT8/P18/P1lyP0E/Pz8/Pyk/SyY/2a0/Pz87OD8/Pz8/P3U/Pz9QP242Iio217c/P2I/YklLP2YmPz8/Mig/Kz8/ZlwxP3pdPyU/WMugP0gvPz8/P0U/Pz9AUMyjP1c/Pzc/RT8/UG4/Pz8/Pz84K08/Pz9mej8qJj8/0p8/4oKsP04/TykkPz8/e2Q/aD887428Pz/Ut1plVD8/Pz8/Pz8nPz8/Ij8/P1MNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9OPz8/Py16OT8/1bEzPyc/ZDkxPz8/P9KrPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9vYz8/RDLVkExCPz9BPyE/P2E/yLk/eD9oPz9HP0jMnz8/Pz9RPz8/Pz8/Mj960rM/fT8kJz8/Pz8/P0o/Pz80Lz8/Pz9AVT8/Pz9kPz8/Pz8/Xnw/bj8/QdeXPz9TPzw/Pz96Z3l6P195Pz93PzE/NT8/Pz8/Pzk/w5ZzPz8uK28/Pz9ZSz8jPz8/1Jg/Pz9nP1lPPz8/DQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBlPz8yPz92P9anPT8/Pz8/PCU/PD9mP1lxPzk/NDtbzoQ/XD9XPz8/PzHanD8/OnTKhT8wPz8/P1w/Pz9uP0w/P3Q/ZT8/Pz8/Py10Pz8uPz8/P2QpYT8/Pzw/Pz9vdD9dDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA7P0JRJlTegT9nZjk/I3bNh0A/P0JGOD9qZj9uPyVLP1R9eH5VPz8/Pz8tNSE/Yz8/cSs/Zj90NGZmbUEpPyM/PyI/dj8tWz4/Pz9+PNSuPz8/P3INCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9mPz8/Pz9+Pz8/Pz8/Jz8/Py4/Pz9NID8/Pz9idgk/P1I/Pyo/xIg/dl8/Pz/Vtk4/P2I/e2s/PzA/P0JVP2E/ID8rPz8hQVE/NT18P0FgIj8/fG4/TD8/Pz9NeS4oPz8/P3IqPz8/JUkiPz89LV4/TT8/Iz8/Pz8/P34/ME3Ht8mpPz8/RT8/cj8/P2Q/UC5LPz8/bCI/bFBdPz8kP0M/Pz9KCX0NCk4/Pz8/Pzk/bEc/Pz9szII/aVU3ICY/RChsPz9IPz8/bT8/P1cjP8yoPyvbmUo/eTo3P0A/fFd9P2E/Pz90eWg/P1o/Pz8/YT8/ej8pPz8/Pz9tPzQlPzc/0rVa2IU/Sz8/WD9+06c/Pz8/Pz8/Pz/RtT8/PzUzKjnYrz/XiQ0KRCo/NyNrP3cpP8mrUj8NCj8/Pz8/byc/dyd4Pz87eD8/NT8/Pz8vPz/Yo0FEMj8/UD8/Pz8/PyNMWj8uPz9Qw6k/MjQ/Pz9eOj/Sqj8/P1Y/240/Py15Zz8/bz8/PyQ/Pw0KLz98U1w/OVQ/Pz85Xz8/Pz9aPz8/yLEhcD90VT8/Rj8hy7RFPz9YP2NJVT8leD8/PzM/P1lYOnc/MD8/P3c/Vz8yDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKg0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/06M/Pyc/J2A/PyR7Pz8/P2kwP0JKP18/TGQ/eVk/Pz8/Pz8JP1s/Ud6GeChc1pg2Pz8iP1AlPyo/Pz8tPz9oPz9uPz8/Pj8/Pz8/Pz81LjM/Pzk/Pz89Pz9PPz9GJT9DKOODrj8/bT8/RU4/VXVQJWNvPz86P0p507k/Pzg/P1U/Pz95LFpVPz8/P2A/Oz8/Pz/LnV4jd2I6ac2wP2JRPwk/bj8/26k/WD8wPz8jaT9vPy4/QT8/NkIhPz/fjyU/wpE/cz8/P27dtT95OD8sP2prP3w/P3s/Vj88P9a3P9+KPy4/P2BAdlbSgj8/Imo/P0c/UD9iP250P9aDPz8/PyM/Jkg/Py4/Rz9oRD8/xqgiPz8lPyU/Pz8/RzM/Pz97P3c3Pz/dk0cnPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgP3c/Pz8/P0liYD9ZPzo/JlZtcj8/c2lYPz8/P24/P18/Pz80ank/P0tFKj/OmFJ4P3g/OyMyM2QkNT8/cCsyIj9cPzw9JMmnP1lHPz8/X00lPz9lQnxKTEJp1oI/Nj9D1Z0/VHZWXz9yPz8/cz/apEw/w7JcWz8/Pz98bWw/CT8/P2M/PzM/Pz8/Pz9tPyciPz4/e3bUmQ0KICAgICAgICAgICAgICAgP0w/P0p1Mj9iP2A/aT9mPzw/P2BfPz9eP2FVPz/Yqj8hP0s/P1dgPz9AP0QJP08/fj8/UH4/Pz9nVD8/Pz9uZT9vP+eShD8/Pz4/P3skPz90dmo/cz8jNz9LPyY2P9alP2o/P92OP3g/Jm9jPy8/cj8/Pz8lPz8/P3A+Rz8lP0gqJj8/Pz80PyFRQz8/P8+oPz9kRT8/Pz5RPyE/VFNBSlw/YnQ/P1wgP3E4PyE/P9SMPz8/ZD8/Kg0KPz8obD9ZPyo/P3VTP11GKT8/Pys/Pz8/Pz8/34rXsnFJWz93Pz9aRT9JMzhXTlk/P3hzPz8/xJR8Pz9dSVU4Ljc/Py8/Pz8/Pz8/P2k/0Ig/PwlZeHdoPz8/UT9iWdCEP3k/Pz9IfT9PPz8/TtqBPz/Xu2A/Pz8/ID8/P1Y/DQo/P0lMRj8/Rzo7I0Y2Jj92Kj9HVzw/P0FIPz9vMD8/MmE/Pz8/P1MpVD8/Pz98S1U/Pz9UCTc/ej8/Pz/CtnVJPz9OSnI/Pz/crD8iPT8/Qj9xPz9SLj/voq0NCiAgICAgICAgICBmPz9xPzE/MT8JPz8/Pz8+JHw/Lj9qdT9zYHA/cVM/1ZEJP24/ZT8/Pz9Odjo/y7U/Pz8/LD8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA9XD8/JT8/Pz9YP8uEP1E/Pz/HjT8/V3TVkD8/dz8/fD8/Pz8tOGY0Pz/euj98Pz97aT9XQSk/Pz8/PysnPz8nP0A6TdeBPz85UW91Pyc/UzVlRD9UNz8/P0E/Pz9tZHY/PyI/cD9yT2hGPyRJP9+rMj8lWWA/Pz8/Pz9hdXk/zZcqVD9HPj8/Pz8/XT9kMD9ZND44XT8/ZT8/Q1g/Pz8/SD/YkVk/1JQ/PzM/Oz98Pz92TT8/ej8/P+mqmz8/P3U/Pz/csz9BND8/Pz8/aD8/ZT/Rm8iJPz8/Pz9qPz8/PyhXZD8/P2lCUVA/IEQ/Pz/nlrc/Pz/Viz8/P0NcNT8vQXY/ZnA/Pz9vQD8/Pz8/P29EJD8/Pzc/Pz9YdT8rDQogICAgICAgICAgICAgIO2GsVg/UT87XFE/Pz8/Wz8/2Kc/2Jc/dw0Kdj94Uz8/YD8lP28/Pz80Pz9aRD8yP2ZjPz8uP0R6Pz8/VW8/Xj4/Jj8/dT8/SD/aoz8/Jg0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfj8/Pz8/PXo/Z0dUPz9iPz8/czwnPz9tP2t4P1I/I3k/Nj8/Pz8/RlA0PD8/QT9oPz8sPz97MT8/RGdFcEY/Pz8JPz91Jj8/P2523r02Pz8/P2w3P1VPdD8qP2U/Pz9XcT8/Xj8/K1d6Pz8xP20/woEoXFpCP3Y9dC9NKD8/JD9fPzU/P0x+Pz8/Pz8tRFxSVj993JI/PzQ/Pw0KPz8NCj8hPz8/dyjJgz8/Pz86Nz8ye0c/Pyg/Zj8/LE0/Pz8/MD9yaktHWNyEMFU/fT8/P9OLPzM/bD9wP3cNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/PWU2Pz8/MmZSPyIraA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgS1A/Jz8/XSYoPz8/dj9AbD8/P3J4P3ZoUnJjP1QlPz9F3rxrPz8rPz8/Nj9SaT46Zz8/Pz8zPz8/Pz8/X1Y/P2ZYPz8/Pz9sP++sryU/Pz8kMT8NCiIyPz8/NUdKPz9PNj8/PzE/T3c/Pz8NCmhRSD8/KWU0Pz8/PVw/TD8vPyFBPz0/Xj8kPz9lPz8/Pz9yPmthP08/ajM/KT8/Pz8/ZUE/27g/Pz8hPy8/XW8/Yj9HP0Mjbms/75qFPzFMYj9YPz9oPz8/P8KHPz8/OCY/cT9xIQ0KP28/P1k5KCA/SEA/IVw/ciA/Rz8/Pz8rDQogICAgICAgICAgICAgICAgICAgIC4w34MrPz8/LT8zJGk/Pz9EP0E/PzJoPz92P2A/LT97PycyP1hHcj8/P0ghdXF5XmNkP2M/Pz8/PypINtqDMz8zPz8/ST8NCiAgICAgICAgwp0/PycvTSA/UT8/Sz8/Pz8qDQogICAgICAgICAgICAgICAgICAgICAgICAgPw0KOD9sP8WiPz8/P3E8y595Pz8oPz94Pz8/cj9sPz8NCj8/U0E/fHo5P2cvOE1mP2smPz8qOj9Xaz8/Pz8/Qj8/Tj8/Pz/bh14/SD9cMHI/P1w/Pz8/x6A/Pz8/IVI/PzZfPwkkPzNZWSdtPz9SP2BiP2U/P2c6P3VkP18/Pz85fj8/Pyo/aEjfoT8/PzU/P0FPTz80Pz9NcD91PzJoPz9eP3tiYVQ/Pz8/Uz8iYj8/Pz9rP2w/Pz8/Pz9zcD9geU8/Lio/cz8wPz9xPzc/Nd+FRmw/YG0/Pzw/zYU/TWk/Pz8/VD8/Rz8/Pz8/JXZHfT81zac/Pz8/eD8/fUAlPz8/Z1N2bN2NPyMuPz8gP2xpZD8/NFQ/Kz8/Pz8/Pz8/P1Y/Lj8/P1o/PwkhPz8/cz8/RjYNClkNCiA/MT8/fQ0KP20/YXE/L0A/WDtHP2E/Nm0/cD9VZnFaP0k/TD8nUz8/PyU/Pzk/Pz83P05OPz9wP0EkbUo/WD8/PzLNqT8/bD97Kj8/R8yrPz8/yJ4NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgPz8/SD8/SD9xUT8/P0NrPHk/aj8/Ow0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgUTk/cz8/Pz96IV3VuW1hTD8/MT9rPz/Isj8/Pz94PzE/RD99PzU/PyA/Pz8/Pz8/P111Pz8/IU8/UD84TT8/Mj9QPz8/OShDPyE/W3BKPz/JknM/ZT88P1Y/P3Y/RT82Jz8/VT8kP0Y/Vj9QOz8/aj9+Pz8/J1Y/dWQ/b3E/Pz9iPz8/cnE/Pz8/Jz9OP9CTOHvSpT8wP1Fc3oJPbT8nPz1PP2ojPz8nPz8/Pz8tKuSrpj8/OT8lWD8/Pz8/Pz8/Pz9rU+OYnSQkP1E/xrw/PyHMiD9RemY/P9GYPz8/S1E/SjE/Pz8/P3I/bz/WhD9uS3k/NX4/P1nelD8/TT92Pz8/eD8/YTw/P10/Uz8wP31hP3U/Ol1+SzM/cmh5PypWPz9haj92MlQ/Wz8/Pz8/P00/Uz8/Pz8/dEQ2P0trdz8/Pz/fhHINCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFAiPz8/P2s/P2I/UXs/Pz8/Pz8/Wj8/DQpwPz8wPzc/Pz8/PzVRP0LTjj86aj8/Pik/cj8/PzkNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgPyA/Pz/Ltj94P1Rfej98Tj8/P2tVSz9pP3M/bkc/anA/Pz8hP0A3fT8/Pz9uTz8/Pz8/djomPzNnTnY/TD8semQ/P0U/az8md3k/P9mKPz8/P3w/Pz8/Pz8/Pz83Pz8/Pw0KP3M/Pz8/P1U/Pz8/P1Y/P1R3K1zSpio/P3rRkD8rPzM/P0AzWD8/Pyg/dFM/P1I/cz8/Lz9J0Yc/Zj9qU9uUPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbj8/ez8/aD8vPz/Iqgk/Pz8rJkY/RNKIPz9NP0Q7P144Pz9UUD8/Pz8rPz8/Pz98Mj9TPz97Pz8yRT8/KT8/XnNrY3g/Zj8/VT9FQl1dOj9QP3BmPz99P/C5jLFxJT9ucz9fRls/cUbSk1t5MFk/P2dHTz8/PylrWmk/fU5dPz8/SD8/ZWopPz86zJk/Jkk/Pz85P2k/aSE/Xz9wb0c/MnI/Klk/dT8/JMiM17lZUseJPz8/VD9PPz8/JiU+P9OEPzDUkDAoME8/Yz9CCT8/P3M/P0ddPzc8RsWH7qKKOXRjetyWPz9dPz/jvIg/Tgk/Pz8/PyU/P0jOvj8/Pz8/PytDUlE/bD9JRkk/JWd9e25iPzg/IT9QPz8/Pz93Pz8/2L4hK9Ccbj8/Pyk/Pz8/dT8mP0XGrD8/MD9vd1Q/fT/GtEQ/XT9RP1s/fChiPz9YP2M/P3ZXLT8/Uj8+MHg/Zz8/xpBKPy8xPyM/dz9RPz9OP0s6Pz8/NT9+Pz8/P2RHbz8JxIc/PyQ/P08/Uic7ND8/QU1iPz8/J0FfOz8/Sz9eJj8qP0ZpIj9cPz95Vj87SyRbPz8/Pj8/L1E/RS0/P1c/dTs8UDN8OVk/Qz9yPz86PzE/OjskKt2EP9GuPz8/P1/ZkGM/Pz8/Mz8/Z3I/Pz9APz86Pz8xPz8/DQo/IT9QX9yBeCNTQ2A/PUBVP0lyaj8/P8eSPzgtej8/Pzs/P2INCj8kKz8/c2g/Pz9lLiPdhA0KLiUzPz9QP8e9fGdnP0o/Py1gP3pdPz8/eilofUcvPz8/Vz8gP3E/Rz8nOj8/DQogICAgICAgICAgICAgICAgICAgICAgICYxPz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgID9kcT8/cj9wYz/GgW5SPz8xTQ0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/Pz8/PzdyPD8lVz9UPz8/Pz9sPykgPy/KmFd2Pz8/Nj8/3Lk/P1x1P3g/Oyc/Pz8/IH5fyJPdsz8yRMafND8/P0A/P2w9Wj8/Pz8vOnY/PzI/Pz94Pz8/2oxAPz9bbj8/Ij8/Pyh+LSk/eCNmPz9cPz8/cD8nfj8/d3YjMz/JsT8/KD8/ZD8lDQo/P0s/1pRuPz8/P0o/Oj1yPz8/PyY/P1F6Pz8zPzk/ez8/Pz8/KlA/Pz9gbT8/MT8NCj8/Pz9sPj9MPz8xP3M/P1c/P18/fWNTP01wP04vPGs/aFZ1Sj9aUj8/Pz8rPz8/P08/eN2yeEg/Rz8/QD9aLj90P3R8Lz80P09YOT8/JUg/P9ytPz/Unz8/Pz8/fj9cPz8/VHk/Pz/WqWU/Pz8/Tz8/aeKAmD8/UD8/dj9hPz8/P+ajsT9sLWxSPz9cPz8/Kj8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHLCikk/P9SwPz86Rj9DPz8/azs/PzR70po/UT98Pz8/RA0KICAgICAgID8/PzA/PyVVP0U/Pz8qMj8/Pzk0Pz8/Lz8/U3A/Pz9IJD/gvIg/QFtxPz8jRD8/Oj8NCmVuZHN0cmVhbQ0KZW5kb2JqDQoNCjggMCBvYmoNCjExODcyDQplbmRvYmoNCg0KOSAwIG9iag0KPDwvVHlwZS9Gb250RGVzY3JpcHRvci9Gb250TmFtZS9CQUFBQUErQ291cmllck5ld1BTTVQNCi9GbGFncyA1DQovRm9udEJCb3hbLTEyMSAtNjc5IDYyMiAxMDIxXS9JdGFsaWNBbmdsZSAwDQovQXNjZW50IDgzMg0KL0Rlc2NlbnQgLTMwMA0KL0NhcEhlaWdodCAxMDIwDQovU3RlbVYgODANCi9Gb250RmlsZTIgNyAwIFINCj4+DQplbmRvYmoNCg0KMTAgMCBvYmoNCjw8L0xlbmd0aCAzMTQvRmlsdGVyL0ZsYXRlRGVjb2RlPj4NCnN0cmVhbQ0KMT8/RT8ycSE/NT8/zpw/yKI/Pz8gUT8/P0x4fXo/Mz8/Pz9jP0U/R20rNlQ/Iz8/Pz8/VT8/P2c/S9+oP1vMvU0/O8SwPz/PqWw/P2c/QD83P3pVPnw/Pz9wPzs/PyhyLDY/P3ZMP3JqPz8/PzU/fj4NCiAgICAgICAgICAgICAgICAgICAgICAgICAgP1XNuT95JT8/d1snP34NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERSPz/mlaA/P3wzPz8/Pz9F3J8m0oA/Ik4zPz8/Sc6FP2o/Lz8/eT90Pz8/Pw0K56SMbVA/PzLZhj8/IEF2Pz8/1o0/Pyk/Pz8/Pz8/Vl0/PT9DP0g/bm12Pz8/DQp0Pz8iPyAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBMJ14/Pw0KICAgICB5Pz8/Xz8/Yj9+Pz9OPy0zP0o/WGloND4/P00/Pz8/CQ0KZW5kc3RyZWFtDQplbmRvYmoNCg0KMTEgMCBvYmoNCjw8L1R5cGUvRm9udC9TdWJ0eXBlL1RydWVUeXBlL0Jhc2VGb250L0JBQUFBQStDb3VyaWVyTmV3UFNNVA0KL0ZpcnN0Q2hhciAwDQovTGFzdENoYXIgMjANCi9XaWR0aHNbNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwDQo2MDAgNjAwIDYwMCA2MDAgNjAwIF0NCi9Gb250RGVzY3JpcHRvciA5IDAgUg0KL1RvVW5pY29kZSAxMCAwIFINCj4+DQplbmRvYmoNCg0KMTIgMCBvYmoNCjw8L0YxIDExIDAgUg0KPj4NCmVuZG9iag0KDQoxMyAwIG9iag0KPDwvRm9udCAxMiAwIFINCi9YT2JqZWN0PDwvVHI0IDQgMCBSPj4NCi9FeHRHU3RhdGU8PC9FR1M1IDUgMCBSPj4NCi9Qcm9jU2V0Wy9QREYvVGV4dC9JbWFnZUMvSW1hZ2VJL0ltYWdlQl0NCj4+DQplbmRvYmoNCg0KMSAwIG9iag0KPDwvVHlwZS9QYWdlL1BhcmVudCA2IDAgUi9SZXNvdXJjZXMgMTMgMCBSL01lZGlhQm94WzAgMCA1OTUgODQyXS9Hcm91cDw8L1MvVHJhbnNwYXJlbmN5L0NTL0RldmljZVJHQi9JIHRydWU+Pi9Db250ZW50cyAyIDAgUj4+DQplbmRvYmoNCg0KNiAwIG9iag0KPDwvVHlwZS9QYWdlcw0KL1Jlc291cmNlcyAxMyAwIFINCi9NZWRpYUJveFsgMCAwIDU5NSA4NDIgXQ0KL0tpZHNbIDEgMCBSIF0NCi9Db3VudCAxPj4NCmVuZG9iag0KDQoxNCAwIG9iag0KPDwvVHlwZS9DYXRhbG9nL1BhZ2VzIDYgMCBSDQovT3BlbkFjdGlvblsxIDAgUiAvWFlaIG51bGwgbnVsbCAwXQ0KL0xhbmcoSj9Abj8pDQo+Pg0KZW5kb2JqDQoNCjE1IDAgb2JqDQo8PC9DcmVhdG9yPEQ3OUM1NUQ1OTdDMDUxMjJCQkE3N0M2NDIyRjg+DQovUHJvZHVjZXI8RDc5QzU1Q0U5N0RCNTEyOUJCQTE3QzY0MjJDNTY2M0VDQ0I1QzM0OTdENURDQjAwQUJEM0IyMkE4NkMxNDlEMz4NCi9DcmVhdGlvbkRhdGUobVlnPz8/YXM/P005P1ZrPz8/TT8pPj4NCmVuZG9iag0KDQoxNiAwIG9iag0KPDwvRmlsdGVyL1N0YW5kYXJkL1YgMi9MZW5ndGggMTI4L1IgMy9PKD98QTptP3AhUz8zP0fRhT8/SD8/P86nVzs/VD9cYikvVSg/PzxbP05WYz/Qvz8/fT8pL1AgLTEwMjg+Pg0KZW5kb2JqDQoNCnhyZWYNCjAgMTcNCjAwMDAwMDAwMDAgNjU1MzUgZiANCjAwMDAwMTM0NzAgMDAwMDAgbiANCjAwMDAwMDAwMTkgMDAwMDAgbiANCjAwMDAwMDAyNzYgMDAwMDAgbiANCjAwMDAwMDAyOTYgMDAwMDAgbiANCjAwMDAwMDA0NzMgMDAwMDAgbiANCjAwMDAwMTM2MTMgMDAwMDAgbiANCjAwMDAwMDA1MTMgMDAwMDAgbiANCjAwMDAwMTI0NzAgMDAwMDAgbiANCjAwMDAwMTI0OTIgMDAwMDAgbiANCjAwMDAwMTI2ODcgMDAwMDAgbiANCjAwMDAwMTMwNzEgMDAwMDAgbiANCjAwMDAwMTMzMTIgMDAwMDAgbiANCjAwMDAwMTMzNDUgMDAwMDAgbiANCjAwMDAwMTM3MTIgMDAwMDAgbiANCjAwMDAwMTM4MDkgMDAwMDAgbiANCjAwMDAwMTM5ODQgMDAwMDAgbiANCnRyYWlsZXINCjw8L1NpemUgMTcvUm9vdCAxNCAwIFINCi9FbmNyeXB0IDE2IDAgUg0KL0luZm8gMTUgMCBSDQovSUQgWyA8NTJBOUE5NEE2MzExODQ3QTk2NkI1NjIxRDc3QTRERDM+DQo8NTJBOUE5NEE2MzExODQ3QTk2NkI1NjIxRDc3QTRERDM+IF0NCi9Eb2NDaGVja3N1bSAvNEE4NTkwMDE4QURDRjVCQjRCQTQ1NDFDQUFDQjFBNEUNCj4+DQpzdGFydHhyZWYNCjE0MTIyDQolJUVPRg==" + hello: "world" + + - do: + indices.refresh: {} + + - do: + search: + index: test + body: + query: + match: + hello: "world" + + - match: { hits.total: 1 } + + - do: + search: + index: test + body: + query: + match: + file1.author: "kimchy" + + - match: { hits.total: 1 } + +#--- +# This test has been disabled as it tries to decode an encoded PDF using BouncyCastle lib +# Elasticsearch security manager does not allow permission java.security.SecurityPermission "insertProvider.BC"; +# See https://github.com/elastic/elasticsearch/pull/13077 +# See https://github.com/elastic/elasticsearch-mapper-attachments/pull/150#issuecomment-134247110 +# +# https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/18 +# Encoded content with https://www.base64encode.org/ +# File1 +# +# +# +# Hello +# +# +# +# +#World +# +# File2 is an encrypted PDF with a password + +#"Multiple Attachments With Encrypted Doc Should Fail": +# +# - do: +# indices.create: +# index: test +# body: +# settings: +# index.mapping.attachment.ignore_errors: false +# mappings: +# doc: +# properties: +# file1: +# type: attachment +# file2: +# type: attachment +# - do: +# cluster.health: +# wait_for_status: yellow +# +# - do: +# catch: /(.)*mapper_parsing_exception(.)*The supplied password does not match either the owner or user password in the document\.(.)*/ +# index: +# index: test +# type: doc +# id: 1 +# body: +# file1: "PCFET0NUWVBFIEhUTUwgUFVCTElDICItLy9XM0MvL0RURCBIVE1MIDQuMDEgVHJhbnNpdGlvbmFsLy9FTiINCiAgICAgICAgImh0dHA6Ly93d3cudzMub3JnL1RSL2h0bWw0L2xvb3NlLmR0ZCI+DQo8aHRtbCBsYW5nPSJmciI+DQo8aGVhZD4NCiAgICA8dGl0bGU+SGVsbG88L3RpdGxlPg0KICAgIDxtZXRhIG5hbWU9ImRhdGUiIGNvbnRlbnQ9IjIwMTItMTEtMzAiPg0KICAgIDxtZXRhIG5hbWU9IkF1dGhvciIgY29udGVudD0ia2ltY2h5Ij4NCiAgICA8bWV0YSBuYW1lPSJLZXl3b3JkcyIgY29udGVudD0iZWxhc3RpY3NlYXJjaCxjb29sLGJvbnNhaSI+DQo8L2hlYWQ+DQo8Ym9keT5Xb3JsZDwvYm9keT4NCjwvaHRtbD4NCg==" +# file2: "JVBERi0xLjQNCiXDpMO8w7bDnw0KMiAwIG9iag0KPDwvTGVuZ3RoIDMgMCBSL0ZpbHRlci9GbGF0ZURlY29kZT4+DQpzdHJlYW0NCjB+az8/Pz8/bDU/Nyw+Pz/Ln1hqZT8uTD9LMS0/Jj9bRDM/SFY/Pz92P2B2P1dxRE5BP2k8Wk0/P01RP3c/PytUP2YmPz9OSD94WT8/RD8/P31AID9JZ9KeYT91Pz8/RD8/PURPP0g/P28/bz8/P3k/UVk/dD8/Pzg/P2tjP3lmPz8/Pz9uP01mfEV6dVwmP095P2osPz14Xyk/P0swbQ0KZW5kc3RyZWFtDQplbmRvYmoNCg0KMyAwIG9iag0KMTg2DQplbmRvYmoNCg0KNCAwIG9iag0KPDwvVHlwZS9YT2JqZWN0DQovU3VidHlwZS9Gb3JtDQovQkJveFsgLTkgNDIwIDYwNCA0MjAuMSBdDQovR3JvdXA8PC9TL1RyYW5zcGFyZW5jeS9DUy9EZXZpY2VSR0IvSyB0cnVlPj4NCi9MZW5ndGggOA0KL0ZpbHRlci9GbGF0ZURlY29kZQ0KPj4NCnN0cmVhbQ0KPz8gPw0K24kNCmVuZHN0cmVhbQ0KZW5kb2JqDQoNCjUgMCBvYmoNCjw8L0NBIDAuNQ0KICAgL2NhIDAuNQ0KPj4NCmVuZG9iag0KDQo3IDAgb2JqDQo8PC9MZW5ndGggOCAwIFIvRmlsdGVyL0ZsYXRlRGVjb2RlL0xlbmd0aDEgMTk5MjA+Pg0Kc3RyZWFtDQo/Rz/Lij9lP1E/UT8/MCs/M0I/N00/P00/DQpfdz87ND8/TD8/QT8/Nz8/wrU/Pz9zPzlPUyQ8XT/Vsj9DP9yOT1ljPz8/Uj8/ITY/Pz9Jaz8/dT8/Pw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB7Pz8/Pz8/P3k/dj8/ICl0Ij9sNz8/Pz8/bWs/P9uuPz9zVSM3Pz8zP1A/P3U/VT8sP1wpPz96P0o/PT8/ID8/OkA/Ii52w6YyP+WHmj8/Pz8/dV0/P2U/KD8/P08/P3FBP25Yc3w4P3U/QHs/Pyx5WD9OY9iEYcu3Pz8/P2JfSz8mPz8/Pz8/QTI/Kig/Pz95Pz8/P20/P04/P2w/DQo/P8mxWD8/Pz8/P1Yva3l0bEs/eVxrP3c/TDZRPz9DIT8/P1s/LD8/P2Y/Oj9mPyAgPy5vP2E/DQo/ej/Qsj8/Pz8/TDZRV2k/Pz8oPz86MT8/bXd4P3IkPz9SWT8/Kz9UIcOBLD8tPz8/Pz9LP2DbnT8/P3w9Pj8/P28/Pz9UYUY/fCxOPT8/P04/cj9kP0I/DQpUUD8/zZs/WT/Zk1YmPz8/XcmQYCI/44WwPz8tP2t8Mis/c1A6Pz9NP08/ez8/T1Y/1KY/P30/Pz8/Pz8/Uj8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzDQo/dz9qPz9QP013Pz9PP3E/fD/Rqz8NCiAgICAgICAgICAgICAgICAgICAgPw0KICAgICAgICAgICAgICAgICAgICAgfEE0P1RoPz8/dmsqPz9tPz8/DQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/Pz8/Pz86Kg0KP966Qk9bPzg/LD8/QT8/P1g/ICAgICAgID8/Wz8/Pz9aP3t5Q3I/fjt8TT8/Yz9Cw7gxRz8/Pz8/1qE/P34/P29DPz9GP3E4P2U/Pz9xPz9rWD9uP1Q/CT8/aduZ66aaPz8oCWc/Sj8/eD8/1IsNCj9MMD8/N3N4P3R+Pz8/JD8/Pz8/Kj8/P1Y/P0NWOj8+Pz9hPz9rP3E/cz8/P9yTWik/Pz/Gjz9zPz/Mvm06YNaXRc6tZT8/dNuePz89SzAoP2o/fT8/ej8/NEA/YD8/Nz9PW1k/LD8vVzA/P2I/PzFRP0I8Pz8NCj8/Rj8vPz8zP8WLPzE/Pz/hp6k/Pz/aq0trIF4/P1lUPz8/Iy5uWT8/RGc/P1g/Pz8/xJs/Pz8/IW0/Pz9GDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/SdyZP3fPsk8/xK1PP3FSP08zPz9cPzJ4Pz5rL0ZKYT8nU3VZCTR1fj81aj9hPz9p34XwuLOLSj/Ogz8/ez8gPz91ZT/Ej2o/P0k/Pz8/ST9lLj8/Tz/EhD9VRXs/OUE/P2N0P3pOXz8/cz8/Pz9oUD8/Pz8/V00/P1soPz8/Nz88NFo/Pz96Pyg/Pz82Pz/GqkI/QD84Uj9RP9uoMnk/LiFBPw0KPzQ/QSo3P3w/1YI/X3w/ID/atT/pm4g/Rj8/Nj9CPj8/PyQ/P2A/Pz9QQFw/Pz8/az/cgj8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/TD8/MD8/Pz96P1w/Pz8/Pz8/Vz9JUiR+Pz9nP3stbz/Sp1I/aHE/P1drP2ljPzQiPz9CND8kID9YPz9uPz9m7JCocj91Lj8/MiE/ND9IPz8/PzM/REg/Pz9dP9ioPyc/Ync/PyM/P0Y4Py0/04g/yKY/dz9XQz8/Pz8oej8NClw/Vc6gPz9xPy8/Pz8wDQogICAgICAgICAgICAgPzFjVz8oPyY1fT8/XFY/TjM/P18/Pz8/QCE/P34/PzZRUz8/Pz8/Pz8/R3RyMj8/CVo/Tj8/Mw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9cPyo/Uj8/Pz8oPz8/Pz8xPyU/Wlw/PzQ/dT87P3o/P1c/KM6PPz/XmT8/Pz85PD8zPz9+Pz8xDQo/P9Wlbz8/YduHLj90djwmPz8tdz9vaD/Trz86Pz8yPz8pJz8/MT8nPwl8PyNT07U6Pz8/NtyHPzI/QT8qPzQxTT8/aj8/Yj8/P2pBWlQ/PzZJNT8/PzrNvT8/ID8/P2M/P1A/DQogICAgICAgICAgICAgICA/Pz82L3vMo1NdLT8lPz8vPz8/P2MxOz9qIyQjwoFkPwk/Pz9kbT8/LTc/dj8/Pz95P35uP3RNP3A35oeRIXw/Mj8/ND8/NEY/Ij/KjDc/cD/Wj30uP1M/IygNCiAgICAgICAgICAgICAgICAgICAgICBOPz8/DQogICAgICAgICAgICAgICAgICAgICAgICAgID08Pz94MD8/IT9KZT8/P20/Pz/hjLE/SD9uP1A/Sig/cVR6CT8/Tz8/P0/TvDk/1aw/Pz8oZj9LP0dAPz8/ez8/IWlCP15fMnY/VFYyOD9IPy4/WD8/ez8/Pz9DP0Y8adGF36w3LD8/cj/Rhj8/X1M6bT8/SD8/X1I/M2gJP2c/bFU2P3c/Pz8/VT8/Pz8/dj8/Pz8/P1h5IUM/biQ/Pz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgID9yPz8/P0rprrsnPz8/KykqTDlTPz8/Pz8/Py8/Qz8/Pz8/WG1lPz8/P3g/P1w/NSUiW34nPz9Ecj8/JX0/SHVfR0k/QUFPP9WzPz8/az8/Lz96QD8/YU0/KD9SYj8/Pz8/PyclRQ0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBZPT8/xKo/Pz9qUnk/UFlLPz8/P3t3XX0/UD/Zgz8/Pz8oPyE/Pz8/MzE/Pw0KICAgICAgICAgdD8/Pz9UPz8/dj94PT8/Pz8/Pz9SPzM/Pz8tKj9uSHk/PzDNkz9yUT8/3ZVaP24/Pz9hP3c/aj/Qgz8gPz9RPy8hdT9mXWclP1g/dHs/PWlgP3o/Qz9tXT9TZD0/cD8lP2g/DQogICAgICAgICAgICAgICAgICAgICAgICA/XD8/Pz8/ZT8/P2LZqmt+Pz96N96KP3Q/Ijk/Pz8/KjM/P3k/U2g/PyBnPz8/SD9rP1k/Pz8/R1A0Pz96P3Q/YENRP+eAjz8/Xj9RP3U/P05KP2NbND9xP9GVP3k/Uj8/Jz9vcj8hPyU/P2BMPz9iYz9RPz8/P3E/cD9kP1Q/Oz8/Pz93Pz8/Pz9dQkouP8+kP3NtP1UNCj92PzlvPz89KHs/PT8/cD9manc/Pz8/Tz8/Pz8/Pz9ZPz8/dnA/TD9+Pz9c1os/Pz8/aX4/JD8/Pz8+Pz8/CT/Vr0k/dz8/a3w/JlYNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgP35SP3U/fCY/QmE/P1ZWPz8/Pz9HKz9FJD9rSj85Pj91d09IP0Q/Pz8/Pz9hPy04Pz8/Pz9wPzE/XkQ3P3N6P1s/K1JTWj0/Pzk/Tj9Qaj9HP3s/Pz9NNA0Kej8/P2h9Pyh6Pzg/OD8/Pz/ejC4/Jz8/Jz9RUj8/QD8/JD8/JFgNCj8/Pw0KICAgPz8/PD8/WEk/P2g/cz9wPD90Pz8/PzRhP3pIRS0/e1U/Pyk/P9SrVnRYJGY/Pz9qPyIvPz9tTH0/Zdy5I1I/Q8KfPz8/Pz9CMT8/ZD9lbz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB1Kz50PtSDdEs/UT8/Pz9tXD8/Pz9PyZExP2FOPj8vNHBuVQ0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA7Pj8qZj88P21aST8/Yj8xQD9oPz8hdT9eP3zMrj8sK21hPz8wMCc/ej9bPz8/XVw/YFpNUCI/aUE/Pz8/7IKRNGpFMz91P1RyPz9wP1BSPz8/Tj9lJH5nPz9wPz8/TkduPz8/U30/cT9SPz9Laz8/L34/P9q2P0kzPz/dpF86Pw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQj9cP04/dD5oaT8/Oz8/amssZz/LtD8/ej/FuiE/P3dmPz8/Pz9Ze3s/Pz8/Lj9wPm8oeG5626hFbX4zQj8/P28uPz8/Pzs/Pz8/CT97Mz8/Sz8iJi9VIj95Pz8/VT8/Pz8NCiAgICAgICAgICAgICAgPz9kTT8/Pz9VKz8/Lj9zP3dTzbtAPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/MSY/Pz9oP0g/Wz9KNz9mPz8/xZk/Mz94MjQ/OGR3Zz8/Jj8/PzB5Sj9DPz8lO8qNbj9vcj9NbT8Jw5BoPz8/TWEsPyE/Pz9sDQp5Kj94Pzc/aloiPz88Xz8/ej9APz8jPz9IPz0/1Lc/UD9XOVs/XD8/Pz8/P0huP3E9RD/Urz8/Pz8/NS1bPz8/bD8/P3tfZj8/PD90NdauPz/cvD8/P0o/XT9IfD8/Ij8kPz8/P24/Pz8nPz8/PzchLHZRPz/Mij8/PA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICo/XnFhP2o/XT8/RDzXlz8/xIQ/P1lGPz8/Rd6eITA/U1Y/P20/Pz9wXj8/MD9ePz83P9KIP2A/Pz81Zj8/Pz85TT8/Wz8/P24/ej8vPz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgMD8/PHJIPz8/P34/PwlTLTA0fHU/P1pMPz8sPz8/zpc3TC0/x6A/PyEyPz8/Pz8/Ij96xa8/Sj9ddz8/Pz8/P3ZjVT8/PyA/cD8/PzpY7ZWOOlc/bD8jQGo/Py4/Pz8/Pz9ePz8/Pz8/Pz9kTjNdRy1mT3t0Pz0/ajkgSDQ/VnI/Pz8/Pz0/P2gnJD98P0lHPz91PztzNT8/Py/KsD98Pz8/Nj9wPz8/UlY/Vz9nPyw/P1p7NT96cyFvPyB0Kz8pP1dFPw0KPz8/Kj8/J8meP2E/Pz8/Pz9AXyFjPz8/PyZkx45lIj9FPz84Pz9vZT8/Pz8/YD8/P28/Pz86PykyPz8rPz9xPz8/Py0/LUhwPz/csG4/P3ghPz8wP0J5P8qjZk8/dEo/Pz9OUj82P2JWLHbKujM/DQogICAgICAgICAgICAgICAgICAgICA/Lm5AYD9Wczs/P0I/P2k/Pz8oP0c/NT8/P18/P1lyP0E/Pz8/Pyk/SyY/2a0/Pz87OD8/Pz8/P3U/Pz9QP242Iio217c/P2I/YklLP2YmPz8/Mig/Kz8/ZlwxP3pdPyU/WMugP0gvPz8/P0U/Pz9AUMyjP1c/Pzc/RT8/UG4/Pz8/Pz84K08/Pz9mej8qJj8/0p8/4oKsP04/TykkPz8/e2Q/aD887428Pz/Ut1plVD8/Pz8/Pz8nPz8/Ij8/P1MNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9OPz8/Py16OT8/1bEzPyc/ZDkxPz8/P9KrPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9vYz8/RDLVkExCPz9BPyE/P2E/yLk/eD9oPz9HP0jMnz8/Pz9RPz8/Pz8/Mj960rM/fT8kJz8/Pz8/P0o/Pz80Lz8/Pz9AVT8/Pz9kPz8/Pz8/Xnw/bj8/QdeXPz9TPzw/Pz96Z3l6P195Pz93PzE/NT8/Pz8/Pzk/w5ZzPz8uK28/Pz9ZSz8jPz8/1Jg/Pz9nP1lPPz8/DQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBlPz8yPz92P9anPT8/Pz8/PCU/PD9mP1lxPzk/NDtbzoQ/XD9XPz8/PzHanD8/OnTKhT8wPz8/P1w/Pz9uP0w/P3Q/ZT8/Pz8/Py10Pz8uPz8/P2QpYT8/Pzw/Pz9vdD9dDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA7P0JRJlTegT9nZjk/I3bNh0A/P0JGOD9qZj9uPyVLP1R9eH5VPz8/Pz8tNSE/Yz8/cSs/Zj90NGZmbUEpPyM/PyI/dj8tWz4/Pz9+PNSuPz8/P3INCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9mPz8/Pz9+Pz8/Pz8/Jz8/Py4/Pz9NID8/Pz9idgk/P1I/Pyo/xIg/dl8/Pz/Vtk4/P2I/e2s/PzA/P0JVP2E/ID8rPz8hQVE/NT18P0FgIj8/fG4/TD8/Pz9NeS4oPz8/P3IqPz8/JUkiPz89LV4/TT8/Iz8/Pz8/P34/ME3Ht8mpPz8/RT8/cj8/P2Q/UC5LPz8/bCI/bFBdPz8kP0M/Pz9KCX0NCk4/Pz8/Pzk/bEc/Pz9szII/aVU3ICY/RChsPz9IPz8/bT8/P1cjP8yoPyvbmUo/eTo3P0A/fFd9P2E/Pz90eWg/P1o/Pz8/YT8/ej8pPz8/Pz9tPzQlPzc/0rVa2IU/Sz8/WD9+06c/Pz8/Pz8/Pz/RtT8/PzUzKjnYrz/XiQ0KRCo/NyNrP3cpP8mrUj8NCj8/Pz8/byc/dyd4Pz87eD8/NT8/Pz8vPz/Yo0FEMj8/UD8/Pz8/PyNMWj8uPz9Qw6k/MjQ/Pz9eOj/Sqj8/P1Y/240/Py15Zz8/bz8/PyQ/Pw0KLz98U1w/OVQ/Pz85Xz8/Pz9aPz8/yLEhcD90VT8/Rj8hy7RFPz9YP2NJVT8leD8/PzM/P1lYOnc/MD8/P3c/Vz8yDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKg0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/06M/Pyc/J2A/PyR7Pz8/P2kwP0JKP18/TGQ/eVk/Pz8/Pz8JP1s/Ud6GeChc1pg2Pz8iP1AlPyo/Pz8tPz9oPz9uPz8/Pj8/Pz8/Pz81LjM/Pzk/Pz89Pz9PPz9GJT9DKOODrj8/bT8/RU4/VXVQJWNvPz86P0p507k/Pzg/P1U/Pz95LFpVPz8/P2A/Oz8/Pz/LnV4jd2I6ac2wP2JRPwk/bj8/26k/WD8wPz8jaT9vPy4/QT8/NkIhPz/fjyU/wpE/cz8/P27dtT95OD8sP2prP3w/P3s/Vj88P9a3P9+KPy4/P2BAdlbSgj8/Imo/P0c/UD9iP250P9aDPz8/PyM/Jkg/Py4/Rz9oRD8/xqgiPz8lPyU/Pz8/RzM/Pz97P3c3Pz/dk0cnPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgP3c/Pz8/P0liYD9ZPzo/JlZtcj8/c2lYPz8/P24/P18/Pz80ank/P0tFKj/OmFJ4P3g/OyMyM2QkNT8/cCsyIj9cPzw9JMmnP1lHPz8/X00lPz9lQnxKTEJp1oI/Nj9D1Z0/VHZWXz9yPz8/cz/apEw/w7JcWz8/Pz98bWw/CT8/P2M/PzM/Pz8/Pz9tPyciPz4/e3bUmQ0KICAgICAgICAgICAgICAgP0w/P0p1Mj9iP2A/aT9mPzw/P2BfPz9eP2FVPz/Yqj8hP0s/P1dgPz9AP0QJP08/fj8/UH4/Pz9nVD8/Pz9uZT9vP+eShD8/Pz4/P3skPz90dmo/cz8jNz9LPyY2P9alP2o/P92OP3g/Jm9jPy8/cj8/Pz8lPz8/P3A+Rz8lP0gqJj8/Pz80PyFRQz8/P8+oPz9kRT8/Pz5RPyE/VFNBSlw/YnQ/P1wgP3E4PyE/P9SMPz8/ZD8/Kg0KPz8obD9ZPyo/P3VTP11GKT8/Pys/Pz8/Pz8/34rXsnFJWz93Pz9aRT9JMzhXTlk/P3hzPz8/xJR8Pz9dSVU4Ljc/Py8/Pz8/Pz8/P2k/0Ig/PwlZeHdoPz8/UT9iWdCEP3k/Pz9IfT9PPz8/TtqBPz/Xu2A/Pz8/ID8/P1Y/DQo/P0lMRj8/Rzo7I0Y2Jj92Kj9HVzw/P0FIPz9vMD8/MmE/Pz8/P1MpVD8/Pz98S1U/Pz9UCTc/ej8/Pz/CtnVJPz9OSnI/Pz/crD8iPT8/Qj9xPz9SLj/voq0NCiAgICAgICAgICBmPz9xPzE/MT8JPz8/Pz8+JHw/Lj9qdT9zYHA/cVM/1ZEJP24/ZT8/Pz9Odjo/y7U/Pz8/LD8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA9XD8/JT8/Pz9YP8uEP1E/Pz/HjT8/V3TVkD8/dz8/fD8/Pz8tOGY0Pz/euj98Pz97aT9XQSk/Pz8/PysnPz8nP0A6TdeBPz85UW91Pyc/UzVlRD9UNz8/P0E/Pz9tZHY/PyI/cD9yT2hGPyRJP9+rMj8lWWA/Pz8/Pz9hdXk/zZcqVD9HPj8/Pz8/XT9kMD9ZND44XT8/ZT8/Q1g/Pz8/SD/YkVk/1JQ/PzM/Oz98Pz92TT8/ej8/P+mqmz8/P3U/Pz/csz9BND8/Pz8/aD8/ZT/Rm8iJPz8/Pz9qPz8/PyhXZD8/P2lCUVA/IEQ/Pz/nlrc/Pz/Viz8/P0NcNT8vQXY/ZnA/Pz9vQD8/Pz8/P29EJD8/Pzc/Pz9YdT8rDQogICAgICAgICAgICAgIO2GsVg/UT87XFE/Pz8/Wz8/2Kc/2Jc/dw0Kdj94Uz8/YD8lP28/Pz80Pz9aRD8yP2ZjPz8uP0R6Pz8/VW8/Xj4/Jj8/dT8/SD/aoz8/Jg0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfj8/Pz8/PXo/Z0dUPz9iPz8/czwnPz9tP2t4P1I/I3k/Nj8/Pz8/RlA0PD8/QT9oPz8sPz97MT8/RGdFcEY/Pz8JPz91Jj8/P2523r02Pz8/P2w3P1VPdD8qP2U/Pz9XcT8/Xj8/K1d6Pz8xP20/woEoXFpCP3Y9dC9NKD8/JD9fPzU/P0x+Pz8/Pz8tRFxSVj993JI/PzQ/Pw0KPz8NCj8hPz8/dyjJgz8/Pz86Nz8ye0c/Pyg/Zj8/LE0/Pz8/MD9yaktHWNyEMFU/fT8/P9OLPzM/bD9wP3cNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/PWU2Pz8/MmZSPyIraA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgS1A/Jz8/XSYoPz8/dj9AbD8/P3J4P3ZoUnJjP1QlPz9F3rxrPz8rPz8/Nj9SaT46Zz8/Pz8zPz8/Pz8/X1Y/P2ZYPz8/Pz9sP++sryU/Pz8kMT8NCiIyPz8/NUdKPz9PNj8/PzE/T3c/Pz8NCmhRSD8/KWU0Pz8/PVw/TD8vPyFBPz0/Xj8kPz9lPz8/Pz9yPmthP08/ajM/KT8/Pz8/ZUE/27g/Pz8hPy8/XW8/Yj9HP0Mjbms/75qFPzFMYj9YPz9oPz8/P8KHPz8/OCY/cT9xIQ0KP28/P1k5KCA/SEA/IVw/ciA/Rz8/Pz8rDQogICAgICAgICAgICAgICAgICAgIC4w34MrPz8/LT8zJGk/Pz9EP0E/PzJoPz92P2A/LT97PycyP1hHcj8/P0ghdXF5XmNkP2M/Pz8/PypINtqDMz8zPz8/ST8NCiAgICAgICAgwp0/PycvTSA/UT8/Sz8/Pz8qDQogICAgICAgICAgICAgICAgICAgICAgICAgPw0KOD9sP8WiPz8/P3E8y595Pz8oPz94Pz8/cj9sPz8NCj8/U0E/fHo5P2cvOE1mP2smPz8qOj9Xaz8/Pz8/Qj8/Tj8/Pz/bh14/SD9cMHI/P1w/Pz8/x6A/Pz8/IVI/PzZfPwkkPzNZWSdtPz9SP2BiP2U/P2c6P3VkP18/Pz85fj8/Pyo/aEjfoT8/PzU/P0FPTz80Pz9NcD91PzJoPz9eP3tiYVQ/Pz8/Uz8iYj8/Pz9rP2w/Pz8/Pz9zcD9geU8/Lio/cz8wPz9xPzc/Nd+FRmw/YG0/Pzw/zYU/TWk/Pz8/VD8/Rz8/Pz8/JXZHfT81zac/Pz8/eD8/fUAlPz8/Z1N2bN2NPyMuPz8gP2xpZD8/NFQ/Kz8/Pz8/Pz8/P1Y/Lj8/P1o/PwkhPz8/cz8/RjYNClkNCiA/MT8/fQ0KP20/YXE/L0A/WDtHP2E/Nm0/cD9VZnFaP0k/TD8nUz8/PyU/Pzk/Pz83P05OPz9wP0EkbUo/WD8/PzLNqT8/bD97Kj8/R8yrPz8/yJ4NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgPz8/SD8/SD9xUT8/P0NrPHk/aj8/Ow0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgUTk/cz8/Pz96IV3VuW1hTD8/MT9rPz/Isj8/Pz94PzE/RD99PzU/PyA/Pz8/Pz8/P111Pz8/IU8/UD84TT8/Mj9QPz8/OShDPyE/W3BKPz/JknM/ZT88P1Y/P3Y/RT82Jz8/VT8kP0Y/Vj9QOz8/aj9+Pz8/J1Y/dWQ/b3E/Pz9iPz8/cnE/Pz8/Jz9OP9CTOHvSpT8wP1Fc3oJPbT8nPz1PP2ojPz8nPz8/Pz8tKuSrpj8/OT8lWD8/Pz8/Pz8/Pz9rU+OYnSQkP1E/xrw/PyHMiD9RemY/P9GYPz8/S1E/SjE/Pz8/P3I/bz/WhD9uS3k/NX4/P1nelD8/TT92Pz8/eD8/YTw/P10/Uz8wP31hP3U/Ol1+SzM/cmh5PypWPz9haj92MlQ/Wz8/Pz8/P00/Uz8/Pz8/dEQ2P0trdz8/Pz/fhHINCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFAiPz8/P2s/P2I/UXs/Pz8/Pz8/Wj8/DQpwPz8wPzc/Pz8/PzVRP0LTjj86aj8/Pik/cj8/PzkNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgPyA/Pz/Ltj94P1Rfej98Tj8/P2tVSz9pP3M/bkc/anA/Pz8hP0A3fT8/Pz9uTz8/Pz8/djomPzNnTnY/TD8semQ/P0U/az8md3k/P9mKPz8/P3w/Pz8/Pz8/Pz83Pz8/Pw0KP3M/Pz8/P1U/Pz8/P1Y/P1R3K1zSpio/P3rRkD8rPzM/P0AzWD8/Pyg/dFM/P1I/cz8/Lz9J0Yc/Zj9qU9uUPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbj8/ez8/aD8vPz/Iqgk/Pz8rJkY/RNKIPz9NP0Q7P144Pz9UUD8/Pz8rPz8/Pz98Mj9TPz97Pz8yRT8/KT8/XnNrY3g/Zj8/VT9FQl1dOj9QP3BmPz99P/C5jLFxJT9ucz9fRls/cUbSk1t5MFk/P2dHTz8/PylrWmk/fU5dPz8/SD8/ZWopPz86zJk/Jkk/Pz85P2k/aSE/Xz9wb0c/MnI/Klk/dT8/JMiM17lZUseJPz8/VD9PPz8/JiU+P9OEPzDUkDAoME8/Yz9CCT8/P3M/P0ddPzc8RsWH7qKKOXRjetyWPz9dPz/jvIg/Tgk/Pz8/PyU/P0jOvj8/Pz8/PytDUlE/bD9JRkk/JWd9e25iPzg/IT9QPz8/Pz93Pz8/2L4hK9Ccbj8/Pyk/Pz8/dT8mP0XGrD8/MD9vd1Q/fT/GtEQ/XT9RP1s/fChiPz9YP2M/P3ZXLT8/Uj8+MHg/Zz8/xpBKPy8xPyM/dz9RPz9OP0s6Pz8/NT9+Pz8/P2RHbz8JxIc/PyQ/P08/Uic7ND8/QU1iPz8/J0FfOz8/Sz9eJj8qP0ZpIj9cPz95Vj87SyRbPz8/Pj8/L1E/RS0/P1c/dTs8UDN8OVk/Qz9yPz86PzE/OjskKt2EP9GuPz8/P1/ZkGM/Pz8/Mz8/Z3I/Pz9APz86Pz8xPz8/DQo/IT9QX9yBeCNTQ2A/PUBVP0lyaj8/P8eSPzgtej8/Pzs/P2INCj8kKz8/c2g/Pz9lLiPdhA0KLiUzPz9QP8e9fGdnP0o/Py1gP3pdPz8/eilofUcvPz8/Vz8gP3E/Rz8nOj8/DQogICAgICAgICAgICAgICAgICAgICAgICYxPz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgID9kcT8/cj9wYz/GgW5SPz8xTQ0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/Pz8/PzdyPD8lVz9UPz8/Pz9sPykgPy/KmFd2Pz8/Nj8/3Lk/P1x1P3g/Oyc/Pz8/IH5fyJPdsz8yRMafND8/P0A/P2w9Wj8/Pz8vOnY/PzI/Pz94Pz8/2oxAPz9bbj8/Ij8/Pyh+LSk/eCNmPz9cPz8/cD8nfj8/d3YjMz/JsT8/KD8/ZD8lDQo/P0s/1pRuPz8/P0o/Oj1yPz8/PyY/P1F6Pz8zPzk/ez8/Pz8/KlA/Pz9gbT8/MT8NCj8/Pz9sPj9MPz8xP3M/P1c/P18/fWNTP01wP04vPGs/aFZ1Sj9aUj8/Pz8rPz8/P08/eN2yeEg/Rz8/QD9aLj90P3R8Lz80P09YOT8/JUg/P9ytPz/Unz8/Pz8/fj9cPz8/VHk/Pz/WqWU/Pz8/Tz8/aeKAmD8/UD8/dj9hPz8/P+ajsT9sLWxSPz9cPz8/Kj8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHLCikk/P9SwPz86Rj9DPz8/azs/PzR70po/UT98Pz8/RA0KICAgICAgID8/PzA/PyVVP0U/Pz8qMj8/Pzk0Pz8/Lz8/U3A/Pz9IJD/gvIg/QFtxPz8jRD8/Oj8NCmVuZHN0cmVhbQ0KZW5kb2JqDQoNCjggMCBvYmoNCjExODcyDQplbmRvYmoNCg0KOSAwIG9iag0KPDwvVHlwZS9Gb250RGVzY3JpcHRvci9Gb250TmFtZS9CQUFBQUErQ291cmllck5ld1BTTVQNCi9GbGFncyA1DQovRm9udEJCb3hbLTEyMSAtNjc5IDYyMiAxMDIxXS9JdGFsaWNBbmdsZSAwDQovQXNjZW50IDgzMg0KL0Rlc2NlbnQgLTMwMA0KL0NhcEhlaWdodCAxMDIwDQovU3RlbVYgODANCi9Gb250RmlsZTIgNyAwIFINCj4+DQplbmRvYmoNCg0KMTAgMCBvYmoNCjw8L0xlbmd0aCAzMTQvRmlsdGVyL0ZsYXRlRGVjb2RlPj4NCnN0cmVhbQ0KMT8/RT8ycSE/NT8/zpw/yKI/Pz8gUT8/P0x4fXo/Mz8/Pz9jP0U/R20rNlQ/Iz8/Pz8/VT8/P2c/S9+oP1vMvU0/O8SwPz/PqWw/P2c/QD83P3pVPnw/Pz9wPzs/PyhyLDY/P3ZMP3JqPz8/PzU/fj4NCiAgICAgICAgICAgICAgICAgICAgICAgICAgP1XNuT95JT8/d1snP34NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERSPz/mlaA/P3wzPz8/Pz9F3J8m0oA/Ik4zPz8/Sc6FP2o/Lz8/eT90Pz8/Pw0K56SMbVA/PzLZhj8/IEF2Pz8/1o0/Pyk/Pz8/Pz8/Vl0/PT9DP0g/bm12Pz8/DQp0Pz8iPyAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBMJ14/Pw0KICAgICB5Pz8/Xz8/Yj9+Pz9OPy0zP0o/WGloND4/P00/Pz8/CQ0KZW5kc3RyZWFtDQplbmRvYmoNCg0KMTEgMCBvYmoNCjw8L1R5cGUvRm9udC9TdWJ0eXBlL1RydWVUeXBlL0Jhc2VGb250L0JBQUFBQStDb3VyaWVyTmV3UFNNVA0KL0ZpcnN0Q2hhciAwDQovTGFzdENoYXIgMjANCi9XaWR0aHNbNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwDQo2MDAgNjAwIDYwMCA2MDAgNjAwIF0NCi9Gb250RGVzY3JpcHRvciA5IDAgUg0KL1RvVW5pY29kZSAxMCAwIFINCj4+DQplbmRvYmoNCg0KMTIgMCBvYmoNCjw8L0YxIDExIDAgUg0KPj4NCmVuZG9iag0KDQoxMyAwIG9iag0KPDwvRm9udCAxMiAwIFINCi9YT2JqZWN0PDwvVHI0IDQgMCBSPj4NCi9FeHRHU3RhdGU8PC9FR1M1IDUgMCBSPj4NCi9Qcm9jU2V0Wy9QREYvVGV4dC9JbWFnZUMvSW1hZ2VJL0ltYWdlQl0NCj4+DQplbmRvYmoNCg0KMSAwIG9iag0KPDwvVHlwZS9QYWdlL1BhcmVudCA2IDAgUi9SZXNvdXJjZXMgMTMgMCBSL01lZGlhQm94WzAgMCA1OTUgODQyXS9Hcm91cDw8L1MvVHJhbnNwYXJlbmN5L0NTL0RldmljZVJHQi9JIHRydWU+Pi9Db250ZW50cyAyIDAgUj4+DQplbmRvYmoNCg0KNiAwIG9iag0KPDwvVHlwZS9QYWdlcw0KL1Jlc291cmNlcyAxMyAwIFINCi9NZWRpYUJveFsgMCAwIDU5NSA4NDIgXQ0KL0tpZHNbIDEgMCBSIF0NCi9Db3VudCAxPj4NCmVuZG9iag0KDQoxNCAwIG9iag0KPDwvVHlwZS9DYXRhbG9nL1BhZ2VzIDYgMCBSDQovT3BlbkFjdGlvblsxIDAgUiAvWFlaIG51bGwgbnVsbCAwXQ0KL0xhbmcoSj9Abj8pDQo+Pg0KZW5kb2JqDQoNCjE1IDAgb2JqDQo8PC9DcmVhdG9yPEQ3OUM1NUQ1OTdDMDUxMjJCQkE3N0M2NDIyRjg+DQovUHJvZHVjZXI8RDc5QzU1Q0U5N0RCNTEyOUJCQTE3QzY0MjJDNTY2M0VDQ0I1QzM0OTdENURDQjAwQUJEM0IyMkE4NkMxNDlEMz4NCi9DcmVhdGlvbkRhdGUobVlnPz8/YXM/P005P1ZrPz8/TT8pPj4NCmVuZG9iag0KDQoxNiAwIG9iag0KPDwvRmlsdGVyL1N0YW5kYXJkL1YgMi9MZW5ndGggMTI4L1IgMy9PKD98QTptP3AhUz8zP0fRhT8/SD8/P86nVzs/VD9cYikvVSg/PzxbP05WYz/Qvz8/fT8pL1AgLTEwMjg+Pg0KZW5kb2JqDQoNCnhyZWYNCjAgMTcNCjAwMDAwMDAwMDAgNjU1MzUgZiANCjAwMDAwMTM0NzAgMDAwMDAgbiANCjAwMDAwMDAwMTkgMDAwMDAgbiANCjAwMDAwMDAyNzYgMDAwMDAgbiANCjAwMDAwMDAyOTYgMDAwMDAgbiANCjAwMDAwMDA0NzMgMDAwMDAgbiANCjAwMDAwMTM2MTMgMDAwMDAgbiANCjAwMDAwMDA1MTMgMDAwMDAgbiANCjAwMDAwMTI0NzAgMDAwMDAgbiANCjAwMDAwMTI0OTIgMDAwMDAgbiANCjAwMDAwMTI2ODcgMDAwMDAgbiANCjAwMDAwMTMwNzEgMDAwMDAgbiANCjAwMDAwMTMzMTIgMDAwMDAgbiANCjAwMDAwMTMzNDUgMDAwMDAgbiANCjAwMDAwMTM3MTIgMDAwMDAgbiANCjAwMDAwMTM4MDkgMDAwMDAgbiANCjAwMDAwMTM5ODQgMDAwMDAgbiANCnRyYWlsZXINCjw8L1NpemUgMTcvUm9vdCAxNCAwIFINCi9FbmNyeXB0IDE2IDAgUg0KL0luZm8gMTUgMCBSDQovSUQgWyA8NTJBOUE5NEE2MzExODQ3QTk2NkI1NjIxRDc3QTRERDM+DQo8NTJBOUE5NEE2MzExODQ3QTk2NkI1NjIxRDc3QTRERDM+IF0NCi9Eb2NDaGVja3N1bSAvNEE4NTkwMDE4QURDRjVCQjRCQTQ1NDFDQUFDQjFBNEUNCj4+DQpzdGFydHhyZWYNCjE0MTIyDQolJUVPRg==" +# hello: "world" +# +# - do: +# indices.refresh: {} +# +# - do: +# search: +# index: test +# +# - match: { hits.total: 0 } diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/20_search.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/20_search.yaml new file mode 100644 index 00000000000..95d9cef2cbf --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/20_search.yaml @@ -0,0 +1,114 @@ +# Integration tests for Mapper Attachments plugin +# + +setup: + - do: + indices.create: + index: test + body: + mappings: + doc: + properties: + file: + type: attachment + - do: + cluster.health: + wait_for_status: yellow + +--- +# Encoded content with https://www.base64encode.org/ +# +# +# XHTML test document +# +# +# +# +#

+# This document tests the ability of Apache Tika to extract content +# from an XHTML document. +#

+# +# + +"Mapper Attachment Simple": + + - do: + index: + index: test + type: doc + id: 1 + body: + file: "PGh0bWwgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGh0bWwiPg0KPGhlYWQ+DQogICAgPHRpdGxlPlhIVE1MIHRlc3QgZG9jdW1lbnQ8L3RpdGxlPg0KICAgIDxtZXRhIG5hbWU9IkF1dGhvciIgY29udGVudD0iVGlrYSBEZXZlbG9wZXJzIi8+DQogICAgPG1ldGEgaHR0cC1lcXVpdj0icmVmcmVzaCIgY29udGVudD0iNSIvPg0KPC9oZWFkPg0KPGJvZHk+DQo8cD4NCiAgICBUaGlzIGRvY3VtZW50IHRlc3RzIHRoZSBhYmlsaXR5IG9mIEFwYWNoZSBUaWthIHRvIGV4dHJhY3QgY29udGVudA0KICAgIGZyb20gYW4gPGEgaHJlZj0iaHR0cDovL3d3dy53My5vcmcvVFIveGh0bWwxLyI+WEhUTUwgZG9jdW1lbnQ8L2E+Lg0KPC9wPg0KPC9ib2R5Pg0KPC9odG1sPg==" + + - do: + indices.refresh: {} + + - do: + search: + index: test + body: + query: + match: + file.title: "test document" + + - match: { hits.total: 1 } + +--- +# Encoded content with https://www.base64encode.org/ +#Begin +# +#BeforeLimit AfterLimit +# +#Broadway +# +#Nearing the end +# +#End + +"Mapper Attachment ContentLength Limit": + + - do: + index: + index: test + type: doc + id: "withlimit" + body: + file: + _indexed_chars: 20 + _content: "QmVnaW4NCg0KQmVmb3JlTGltaXQgQWZ0ZXJMaW1pdA0KDQpCcm9hZHdheQ0KDQpOZWFyaW5nIHRoZSBlbmQNCg0KRW5k" + + - do: + index: + index: test + type: doc + id: "nolimit" + body: + file: + _indexed_chars: -1 + _content: "QmVnaW4NCg0KQmVmb3JlTGltaXQgQWZ0ZXJMaW1pdA0KDQpCcm9hZHdheQ0KDQpOZWFyaW5nIHRoZSBlbmQNCg0KRW5k" + + - do: + indices.refresh: {} + + - do: + search: + index: test + body: + query: + match: + file.content: "BeforeLimit" + + - match: { hits.total: 2 } + + - do: + search: + index: test + body: + query: + match: + file.content: "AfterLimit" + + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "nolimit" } + diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml new file mode 100644 index 00000000000..170a8bf7382 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml @@ -0,0 +1,61 @@ +# Integration tests for Mapper Attachments plugin +# + +--- +# Encoded content with https://www.base64encode.org/ +# +# +# XHTML test document +# +# +# +# +#

+# This document tests the ability of Apache Tika to extract content +# from an XHTML document. +#

+# +# +"ContentType and Name": + + - do: + indices.create: + index: test + body: + mappings: + doc: + properties: + "file": + "type": "attachment" + "fields": + "content_type": + "store": "yes" + "name": + "store": "yes" + - do: + cluster.health: + wait_for_status: yellow + + - do: + index: + index: test + type: doc + id: 1 + body: + file: + _content: "QmVnaW4NCg0KQmVmb3JlTGltaXQgQWZ0ZXJMaW1pdA0KDQpCcm9hZHdheQ0KDQpOZWFyaW5nIHRoZSBlbmQNCg0KRW5k" + _content_type: "text/my-dummy-content-type" + _name: "my-dummy-name-txt" + + - do: + indices.refresh: {} + + - do: + search: + index: test + body: + fields: [file.content_type,file.name] + + - match: { hits.total: 1 } + - match: { hits.hits.0.fields: { file.content_type: ["text/my-dummy-content-type"], file.name: ["my-dummy-name-txt"] }} + diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml new file mode 100644 index 00000000000..286dae8b976 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml @@ -0,0 +1,67 @@ +# Integration tests for Mapper Attachments plugin +# + +setup: + - do: + indices.create: + index: test + body: + mappings: + doc: + properties: + "file": + "type": "attachment" + "fields": + "content" : + "type": "string" + "store" : "yes" + "term_vector": "with_positions_offsets" + + - do: + cluster.health: + wait_for_status: yellow + +--- +# Encoded content with https://www.base64encode.org/ +# +# +# XHTML test document +# +# +# +# +#

+# This document tests the ability of Apache Tika to extract content +# from an XHTML document. +#

+# +# + +"Highlight content": + + - do: + index: + index: test + type: doc + id: 1 + body: + file: "PGh0bWwgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGh0bWwiPg0KPGhlYWQ+DQogICAgPHRpdGxlPlhIVE1MIHRlc3QgZG9jdW1lbnQ8L3RpdGxlPg0KICAgIDxtZXRhIG5hbWU9IkF1dGhvciIgY29udGVudD0iVGlrYSBEZXZlbG9wZXJzIi8+DQogICAgPG1ldGEgaHR0cC1lcXVpdj0icmVmcmVzaCIgY29udGVudD0iNSIvPg0KPC9oZWFkPg0KPGJvZHk+DQo8cD4NCiAgICBUaGlzIGRvY3VtZW50IHRlc3RzIHRoZSBhYmlsaXR5IG9mIEFwYWNoZSBUaWthIHRvIGV4dHJhY3QgY29udGVudA0KICAgIGZyb20gYW4gPGEgaHJlZj0iaHR0cDovL3d3dy53My5vcmcvVFIveGh0bWwxLyI+WEhUTUwgZG9jdW1lbnQ8L2E+Lg0KPC9wPg0KPC9ib2R5Pg0KPC9odG1sPg==" + + - do: + indices.refresh: {} + + - do: + search: + index: test + body: + query: + match: + file.content: "apache tika" + fields: [] + highlight: + fields: + file.content: {} + + - match: { hits.total: 1 } + - match: { hits.hits.0.highlight: { file.content : [ "\n\n This document tests the ability of Apache Tika to extract content\n from an XHTML document.\n" ] }} + diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 932726d1871..a12740ce8c6 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -23,9 +23,8 @@ esplugin { } dependencies { - compile('com.microsoft.azure:azure-storage:2.0.0') { - exclude group: 'org.slf4j', module: 'slf4j-api' - } + compile 'com.microsoft.azure:azure-storage:2.0.0' + compile 'org.apache.commons:commons-lang3:3.3.2' } dependencyLicenses { diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java index a7c980c8159..c10abb114b6 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java @@ -23,7 +23,9 @@ import com.microsoft.azure.storage.StorageException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.BlobPath; -import org.elasticsearch.common.blobstore.support.AbstractLegacyBlobContainer; +import org.elasticsearch.common.blobstore.support.AbstractBlobContainer; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.repositories.RepositoryException; @@ -39,7 +41,7 @@ import java.util.Map; /** * */ -public class AzureBlobContainer extends AbstractLegacyBlobContainer { +public class AzureBlobContainer extends AbstractBlobContainer { protected final ESLogger logger = Loggers.getLogger(AzureBlobContainer.class); protected final AzureBlobStore blobStore; @@ -69,7 +71,7 @@ public class AzureBlobContainer extends AbstractLegacyBlobContainer { } @Override - public InputStream openInput(String blobName) throws IOException { + public InputStream readBlob(String blobName) throws IOException { try { return blobStore.client().getInputStream(blobStore.container(), buildKey(blobName)); } catch (StorageException e) { @@ -83,7 +85,20 @@ public class AzureBlobContainer extends AbstractLegacyBlobContainer { } @Override - public OutputStream createOutput(String blobName) throws IOException { + public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException { + try (OutputStream stream = createOutput(blobName)) { + Streams.copy(inputStream, stream); + } + } + + @Override + public void writeBlob(String blobName, BytesReference bytes) throws IOException { + try (OutputStream stream = createOutput(blobName)) { + bytes.writeTo(stream); + } + } + + private OutputStream createOutput(String blobName) throws IOException { try { return new AzureOutputStream(blobStore.client().getOutputStream(blobStore.container(), buildKey(blobName))); } catch (StorageException e) { diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 9fb09afa2a9..32ad37530c2 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -22,9 +22,20 @@ esplugin { classname 'org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin' } +versions << [ + 'aws': '1.10.33' +] + dependencies { - compile 'com.amazonaws:aws-java-sdk-s3:1.10.19' + compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}" + compile "com.amazonaws:aws-java-sdk-kms:${versions.aws}" + compile "com.amazonaws:aws-java-sdk-core:${versions.aws}" compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" + compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" + compile "commons-logging:commons-logging:${versions.commonslogging}" + compile "commons-codec:commons-codec:${versions.commonscodec}" + compile "com.fasterxml.jackson.core:jackson-databind:2.5.3" + compile "com.fasterxml.jackson.core:jackson-annotations:2.5.0" } dependencyLicenses { diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.19.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.19.jar.sha1 deleted file mode 100644 index 66e418e6fb2..00000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.19.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b53f650323b7242dcced25b679f3e9aa4b494da5 diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.33.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.33.jar.sha1 new file mode 100644 index 00000000000..332a8f01035 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.33.jar.sha1 @@ -0,0 +1 @@ +fabedbbe2b834b1add150b6a38395c5ef7380168 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.19.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.19.jar.sha1 deleted file mode 100644 index 1328451c1c0..00000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.19.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c8764f3e61a3c420db429870ec22b31fe755d81d diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.33.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.33.jar.sha1 new file mode 100644 index 00000000000..0d7ab9f8381 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.33.jar.sha1 @@ -0,0 +1 @@ +35881245894ecc4d893c074eacdf2e6b56820fda \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.19.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.19.jar.sha1 deleted file mode 100644 index 9932c4676c4..00000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.19.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a23dc60d56d54126250c23cab1d01328b1e83678 diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.33.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.33.jar.sha1 new file mode 100644 index 00000000000..3328f01c658 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.33.jar.sha1 @@ -0,0 +1 @@ +5665cf77102a932a16e99ebf41d197e03ddbf25c \ No newline at end of file diff --git a/plugins/repository-s3/licenses/commons-codec-1.10.jar.sha1 b/plugins/repository-s3/licenses/commons-codec-1.10.jar.sha1 new file mode 100644 index 00000000000..3fe8682a1b0 --- /dev/null +++ b/plugins/repository-s3/licenses/commons-codec-1.10.jar.sha1 @@ -0,0 +1 @@ +4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/commons-codec-1.6.jar.sha1 b/plugins/repository-s3/licenses/commons-codec-1.6.jar.sha1 deleted file mode 100644 index bf78aff7364..00000000000 --- a/plugins/repository-s3/licenses/commons-codec-1.6.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b7f0fc8f61ecadeb3695f0b9464755eee44374d4 diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java index 09194b2ccab..4f5c46a8c59 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java @@ -25,9 +25,11 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStoreException; -import org.elasticsearch.common.blobstore.support.AbstractLegacyBlobContainer; +import org.elasticsearch.common.blobstore.support.AbstractBlobContainer; import org.elasticsearch.common.blobstore.support.PlainBlobMetaData; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.io.Streams; import java.io.FileNotFoundException; import java.io.IOException; @@ -38,7 +40,7 @@ import java.util.Map; /** * */ -public class S3BlobContainer extends AbstractLegacyBlobContainer { +public class S3BlobContainer extends AbstractBlobContainer { protected final S3BlobStore blobStore; @@ -67,16 +69,7 @@ public class S3BlobContainer extends AbstractLegacyBlobContainer { } @Override - public void deleteBlob(String blobName) throws IOException { - try { - blobStore.client().deleteObject(blobStore.bucket(), buildKey(blobName)); - } catch (AmazonClientException e) { - throw new IOException("Exception when deleting blob [" + blobName + "]", e); - } - } - - @Override - public InputStream openInput(String blobName) throws IOException { + public InputStream readBlob(String blobName) throws IOException { int retry = 0; while (retry <= blobStore.numberOfRetries()) { try { @@ -99,9 +92,32 @@ public class S3BlobContainer extends AbstractLegacyBlobContainer { } @Override - public OutputStream createOutput(final String blobName) throws IOException { + public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException { + try (OutputStream stream = createOutput(blobName)) { + Streams.copy(inputStream, stream); + } + } + + @Override + public void writeBlob(String blobName, BytesReference bytes) throws IOException { + try (OutputStream stream = createOutput(blobName)) { + bytes.writeTo(stream); + } + } + + @Override + public void deleteBlob(String blobName) throws IOException { + try { + blobStore.client().deleteObject(blobStore.bucket(), buildKey(blobName)); + } catch (AmazonClientException e) { + throw new IOException("Exception when deleting blob [" + blobName + "]", e); + } + } + + private OutputStream createOutput(final String blobName) throws IOException { // UploadS3OutputStream does buffering & retry logic internally - return new DefaultS3OutputStream(blobStore, blobStore.bucket(), buildKey(blobName), blobStore.bufferSizeInBytes(), blobStore.numberOfRetries(), blobStore.serverSideEncryption()); + return new DefaultS3OutputStream(blobStore, blobStore.bucket(), buildKey(blobName), + blobStore.bufferSizeInBytes(), blobStore.numberOfRetries(), blobStore.serverSideEncryption()); } @Override diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java index d014613dc3e..2911e278c38 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java @@ -19,7 +19,6 @@ package org.elasticsearch.plugin.repository.s3; -import org.elasticsearch.SpecialPermission; import org.elasticsearch.cloud.aws.S3Module; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Module; @@ -28,8 +27,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.RepositoriesModule; import org.elasticsearch.repositories.s3.S3Repository; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -38,27 +35,6 @@ import java.util.Collections; * */ public class S3RepositoryPlugin extends Plugin { - - static { - // This internal config is deserialized but with wrong access modifiers, - // cannot work without suppressAccessChecks permission right now. We force - // a one time load with elevated privileges as a workaround. - SecurityManager sm = System.getSecurityManager(); - if (sm != null) { - sm.checkPermission(new SpecialPermission()); - } - AccessController.doPrivileged(new PrivilegedAction() { - @Override - public Void run() { - try { - Class.forName("com.amazonaws.internal.config.InternalConfig$Factory"); - } catch (ClassNotFoundException e) { - throw new RuntimeException("Unable to initialize internal aws config", e); - } - return null; - } - }); - } @Override public String name() { diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java index 846892b8704..0c9e7535db0 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java @@ -628,4 +628,9 @@ public class AmazonS3Wrapper implements AmazonS3 { public BucketReplicationConfiguration getBucketReplicationConfiguration(GetBucketReplicationConfigurationRequest getBucketReplicationConfigurationRequest) throws AmazonServiceException, AmazonClientException { return delegate.getBucketReplicationConfiguration(getBucketReplicationConfigurationRequest); } + + @Override + public HeadBucketResult headBucket(HeadBucketRequest headBucketRequest) throws AmazonClientException, AmazonServiceException { + return delegate.headBucket(headBucketRequest); + } } diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java index a375fda16ba..23dae876210 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java @@ -632,6 +632,7 @@ public class PluginManagerTests extends ESIntegTestCase { PluginManager.checkForOfficialPlugins("lang-groovy"); PluginManager.checkForOfficialPlugins("lang-javascript"); PluginManager.checkForOfficialPlugins("lang-python"); + PluginManager.checkForOfficialPlugins("mapper-attachments"); PluginManager.checkForOfficialPlugins("mapper-murmur3"); PluginManager.checkForOfficialPlugins("mapper-size"); PluginManager.checkForOfficialPlugins("discovery-multicast"); diff --git a/run.bat b/run.bat deleted file mode 100755 index b0cc48a88ab..00000000000 --- a/run.bat +++ /dev/null @@ -1,5 +0,0 @@ -:: -:: build zip package, but ensuring its from the current source -:: turn off tests and other validation to speed it up -:: TODO: can be sped up more, if shading is moved out of core/ -CALL mvn -am -pl dev-tools,distribution/zip package -DskipTests -Drun -Pdev diff --git a/run.sh b/run.sh deleted file mode 100755 index 16871065918..00000000000 --- a/run.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -# -# build zip package, but ensuring its from the current source -# turn off tests and other validation to speed it up -# TODO: can be sped up more, if shading is moved out of core/ -mvn -am -pl dev-tools,distribution/zip package -DskipTests -Drun -Pdev diff --git a/settings.gradle b/settings.gradle index 2cd793703a4..0d2828d221a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -23,6 +23,7 @@ List projects = [ 'plugins:lang-groovy', 'plugins:lang-javascript', 'plugins:lang-python', + 'plugins:mapper-attachments', 'plugins:mapper-murmur3', 'plugins:mapper-size', 'plugins:repository-azure', diff --git a/test-framework/build.gradle b/test-framework/build.gradle index a1d96a77f6f..8252df9e3dc 100644 --- a/test-framework/build.gradle +++ b/test-framework/build.gradle @@ -24,16 +24,14 @@ apply plugin: 'com.bmuschko.nexus' dependencies { compile "org.elasticsearch:elasticsearch:${version}" compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" - compile(group: 'junit', name: 'junit', version: '4.11') { - exclude group: 'org.hamcrest', module: 'hamcrest-core' - } - compile("org.apache.lucene:lucene-test-framework:${versions.lucene}") { - exclude group: 'com.carrotsearch.randomizedtesting', module: 'junit4-ant' - } - compile('org.hamcrest:hamcrest-all:1.3') { - exclude group: 'org.hamcrest', module: 'hamcrest-core' - } + compile "junit:junit:${versions.junit}" + compile 'org.hamcrest:hamcrest-all:1.3' + compile "org.apache.lucene:lucene-test-framework:${versions.lucene}" + compile "org.apache.lucene:lucene-codecs:${versions.lucene}" compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" + compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" + compile "commons-logging:commons-logging:${versions.commonslogging}" + compile "commons-codec:commons-codec:${versions.commonscodec}" } compileJava.options.compilerArgs << '-Xlint:-cast,-deprecation,-fallthrough,-overrides,-rawtypes,-serial,-try,-unchecked' diff --git a/test-framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java b/test-framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java index d12f4d68752..767b1683f2c 100644 --- a/test-framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java +++ b/test-framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java @@ -36,6 +36,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.Decision; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.gateway.GatewayAllocator; import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.test.gateway.NoopGatewayAllocator; @@ -79,6 +80,12 @@ public abstract class ESAllocationTestCase extends ESTestCase { new ShardsAllocators(settings, NoopGatewayAllocator.INSTANCE), clusterInfoService); } + public static AllocationService createAllocationService(Settings settings, GatewayAllocator allocator) { + return new AllocationService(settings, + randomAllocationDeciders(settings, new NodeSettingsService(Settings.Builder.EMPTY_SETTINGS), getRandom()), + new ShardsAllocators(settings, allocator), EmptyClusterInfoService.INSTANCE); + } + public static AllocationDeciders randomAllocationDeciders(Settings settings, NodeSettingsService nodeSettingsService, Random random) { diff --git a/test-framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java b/test-framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java index ac582511032..825b203022d 100644 --- a/test-framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java +++ b/test-framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java @@ -32,7 +32,7 @@ public class NoopGatewayAllocator extends GatewayAllocator { public static final NoopGatewayAllocator INSTANCE = new NoopGatewayAllocator(); - private NoopGatewayAllocator() { + protected NoopGatewayAllocator() { super(Settings.EMPTY, null, null); }