mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-03-25 01:19:02 +00:00
This PR adds a new 'version' field type that allows indexing string values representing software versions similar to the ones defined in the Semantic Versioning definition (semver.org). The field behaves very similar to a 'keyword' field but allows efficient sorting and range queries that take into accound the special ordering needed for version strings. For example, the main version parts are sorted numerically (ie 2.0.0 < 11.0.0) whereas this wouldn't be possible with 'keyword' fields today. Valid version values are similar to the Semantic Versioning definition, with the notable exception that in addition to the "main" version consiting of major.minor.patch, we allow less or more than three numeric identifiers, i.e. "1.2" or "1.4.6.123.12" are treated as valid too. Relates to #48878
This commit is contained in:
Christoph Büscher
GitHub
parent
5e0f9a414c
commit
803f78ef05
@ -51,6 +51,8 @@ Dates:: Date types, including <<date,`date`>> and
|
||||
<<range,Range>>:: Range types, such as `long_range`, `double_range`,
|
||||
`date_range`, and `ip_range`.
|
||||
<<ip,`ip`>>:: IPv4 and IPv6 addresses.
|
||||
<<version,Version>>:: Software versions. Supports https://semver.org/[Semantic Versioning]
|
||||
precedence rules.
|
||||
{plugins}/mapper-murmur3.html[`murmur3`]:: Compute and stores hashes of
|
||||
values.
|
||||
|
||||
@ -148,6 +150,8 @@ include::types/geo-shape.asciidoc[]
|
||||
|
||||
include::types/ip.asciidoc[]
|
||||
|
||||
include::types/version.asciidoc[]
|
||||
|
||||
include::types/parent-join.asciidoc[]
|
||||
|
||||
include::types/keyword.asciidoc[]
|
||||
|
||||
@ -134,3 +134,4 @@ The following parameters are accepted by `keyword` fields:
|
||||
include::constant-keyword.asciidoc[]
|
||||
|
||||
include::wildcard.asciidoc[]
|
||||
|
||||
|
||||
70
docs/reference/mapping/types/version.asciidoc
Normal file
70
docs/reference/mapping/types/version.asciidoc
Normal file
@ -0,0 +1,70 @@
|
||||
[role="xpack"]
|
||||
[testenv="basic"]
|
||||
[[version]]
|
||||
=== Version field type
|
||||
++++
|
||||
<titleabbrev>Version</titleabbrev>
|
||||
++++
|
||||
|
||||
The `version` field type is a specialization of the `keyword` field for
|
||||
handling software version values and to support specialized precedence
|
||||
rules for them. Precedence is defined following the rules outlined by
|
||||
https://semver.org/[Semantic Versioning], which for example means that
|
||||
major, minor and patch version parts are sorted numerically (i.e.
|
||||
"2.1.0" < "2.4.1" < "2.11.2") and pre-release versions are sorted before
|
||||
release versions (i.e. "1.0.0-alpha" < "1.0.0").
|
||||
|
||||
You index a `version` field as follows
|
||||
|
||||
[source,console]
|
||||
--------------------------------------------------
|
||||
PUT my-index-000001
|
||||
{
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"my_version": {
|
||||
"type": "version"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
--------------------------------------------------
|
||||
|
||||
The field offers the same search capabilities as a regular keyword field. It
|
||||
can e.g. be searched for exact matches using `match` or `term` queries and
|
||||
supports prefix and wildcard searches. The main benefit is that `range` queries
|
||||
will honor Semver ordering, so a `range` query between "1.0.0" and "1.5.0"
|
||||
will include versions of "1.2.3" but not "1.11.2" for example. Note that this
|
||||
would be different when using a regular `keyword` field for indexing where ordering
|
||||
is alphabetical.
|
||||
|
||||
Software versions are expected to follow the
|
||||
https://semver.org/[Semantic Versioning rules] schema and precedence rules with
|
||||
the notable exception that more or less than three main version identifiers are
|
||||
allowed (i.e. "1.2" or "1.2.3.4" qualify as valid versions while they wouldn't under
|
||||
strict Semver rules). Version strings that are not valid under the Semver definition
|
||||
(e.g. "1.2.alpha.4") can still be indexed and retrieved as exact matches, however they
|
||||
will all appear _after_ any valid version with regular alphabetical ordering. The empty
|
||||
String "" is considered invalid and sorted after all valid versions, but before other
|
||||
invalid ones.
|
||||
|
||||
[discrete]
|
||||
[[version-params]]
|
||||
==== Parameters for version fields
|
||||
|
||||
The following parameters are accepted by `version` fields:
|
||||
|
||||
[horizontal]
|
||||
|
||||
<<mapping-field-meta,`meta`>>::
|
||||
|
||||
Metadata about the field.
|
||||
|
||||
[discrete]
|
||||
==== Limitations
|
||||
|
||||
This field type isn't optimized for heavy wildcard, regex or fuzzy searches. While those
|
||||
type of queries work in this field, you should consider using a regular `keyword` field if
|
||||
you strongly rely on these kind of queries.
|
||||
|
||||
@ -33,9 +33,10 @@ import java.util.Map;
|
||||
|
||||
/** Base {@link MappedFieldType} implementation for a field that is indexed
|
||||
* with the inverted index. */
|
||||
abstract class TermBasedFieldType extends SimpleMappedFieldType {
|
||||
public abstract class TermBasedFieldType extends SimpleMappedFieldType {
|
||||
|
||||
TermBasedFieldType(String name, boolean isSearchable, boolean hasDocValues, TextSearchInfo textSearchInfo, Map<String, String> meta) {
|
||||
public TermBasedFieldType(String name, boolean isSearchable, boolean hasDocValues, TextSearchInfo textSearchInfo,
|
||||
Map<String, String> meta) {
|
||||
super(name, isSearchable, hasDocValues, textSearchInfo, meta);
|
||||
}
|
||||
|
||||
|
||||
@ -101,7 +101,7 @@ public class StringStatsAggregator extends MetricsAggregator {
|
||||
for (int i = 0; i < valuesCount; i++) {
|
||||
BytesRef value = values.nextValue();
|
||||
if (value.length > 0) {
|
||||
String valueStr = value.utf8ToString();
|
||||
String valueStr = (String) format.format(value);
|
||||
int length = valueStr.length();
|
||||
totalLength.increment(bucket, length);
|
||||
|
||||
|
||||
@ -0,0 +1,107 @@
|
||||
# Integration tests for the version field
|
||||
#
|
||||
---
|
||||
setup:
|
||||
|
||||
- skip:
|
||||
features: headers
|
||||
version: " - 7.99.99"
|
||||
reason: "version field is added to 8.0 first"
|
||||
|
||||
- do:
|
||||
indices.create:
|
||||
index: test_index
|
||||
body:
|
||||
mappings:
|
||||
properties:
|
||||
version:
|
||||
type: version
|
||||
|
||||
- do:
|
||||
bulk:
|
||||
refresh: true
|
||||
body:
|
||||
- '{ "index" : { "_index" : "test_index", "_id" : "1" } }'
|
||||
- '{"version": "1.1.0" }'
|
||||
- '{ "index" : { "_index" : "test_index", "_id" : "2" } }'
|
||||
- '{"version": "2.0.0-beta" }'
|
||||
- '{ "index" : { "_index" : "test_index", "_id" : "3" } }'
|
||||
- '{"version": "3.1.0" }'
|
||||
|
||||
---
|
||||
"Store malformed":
|
||||
- do:
|
||||
indices.create:
|
||||
index: test_malformed
|
||||
body:
|
||||
mappings:
|
||||
properties:
|
||||
version:
|
||||
type: version
|
||||
|
||||
- do:
|
||||
bulk:
|
||||
refresh: true
|
||||
body:
|
||||
- '{ "index" : { "_index" : "test_malformed", "_id" : "1" } }'
|
||||
- '{"version": "1.1.0" }'
|
||||
- '{ "index" : { "_index" : "test_malformed", "_id" : "2" } }'
|
||||
- '{"version": "2.0.0-beta" }'
|
||||
- '{ "index" : { "_index" : "test_malformed", "_id" : "3" } }'
|
||||
- '{"version": "v3.1.0" }'
|
||||
- '{ "index" : { "_index" : "test_malformed", "_id" : "4" } }'
|
||||
- '{"version": "1.el6" }'
|
||||
|
||||
- do:
|
||||
search:
|
||||
index: test_malformed
|
||||
body:
|
||||
query: { "match" : { "version" : "1.el6" } }
|
||||
|
||||
- do:
|
||||
search:
|
||||
index: test_malformed
|
||||
body:
|
||||
query: { "match_all" : { } }
|
||||
sort:
|
||||
version: asc
|
||||
|
||||
- match: { hits.total.value: 4 }
|
||||
- match: { hits.hits.0._source.version: "1.1.0" }
|
||||
- match: { hits.hits.1._source.version: "2.0.0-beta" }
|
||||
- match: { hits.hits.2._source.version: "1.el6" }
|
||||
- match: { hits.hits.3._source.version: "v3.1.0" }
|
||||
|
||||
---
|
||||
"Basic ranges":
|
||||
- do:
|
||||
search:
|
||||
index: test_index
|
||||
body:
|
||||
query: { "range" : { "version" : { "gt" : "1.1.0", "lt" : "9999" } } }
|
||||
|
||||
- match: { hits.total.value: 2 }
|
||||
|
||||
- do:
|
||||
search:
|
||||
index: test_index
|
||||
body:
|
||||
query: { "range" : { "version" : { "gte" : "1.1.0", "lt" : "9999" } } }
|
||||
|
||||
- match: { hits.total.value: 3 }
|
||||
|
||||
- do:
|
||||
search:
|
||||
index: test_index
|
||||
body:
|
||||
query: { "range" : { "version" : { "gte" : "2.0.0", "lt" : "9999" } } }
|
||||
|
||||
- match: { hits.total.value: 1 }
|
||||
|
||||
- do:
|
||||
search:
|
||||
index: test_index
|
||||
body:
|
||||
query: { "range" : { "version" : { "gte" : "2.0.0-alpha", "lt" : "9999" } } }
|
||||
|
||||
- match: { hits.total.value: 2 }
|
||||
@ -0,0 +1,54 @@
|
||||
# Integration tests for the version field
|
||||
#
|
||||
---
|
||||
setup:
|
||||
|
||||
- skip:
|
||||
features: headers
|
||||
version: " - 7.99.99"
|
||||
reason: "version field is added to 8.0 first"
|
||||
|
||||
- do:
|
||||
indices.create:
|
||||
index: test_index
|
||||
body:
|
||||
mappings:
|
||||
properties:
|
||||
version:
|
||||
type: version
|
||||
|
||||
- do:
|
||||
bulk:
|
||||
refresh: true
|
||||
body:
|
||||
- '{ "index" : { "_index" : "test_index", "_id" : "1" } }'
|
||||
- '{"version": "1.1.12" }'
|
||||
- '{ "index" : { "_index" : "test_index", "_id" : "2" } }'
|
||||
- '{"version": "2.0.0-beta" }'
|
||||
- '{ "index" : { "_index" : "test_index", "_id" : "3" } }'
|
||||
- '{"version": "3.1.0" }'
|
||||
|
||||
---
|
||||
"Filter script":
|
||||
- do:
|
||||
search:
|
||||
index: test_index
|
||||
body:
|
||||
query: { "script" : { "script" : { "source": "doc['version'].value.length() > 5"} } }
|
||||
|
||||
- match: { hits.total.value: 2 }
|
||||
- match: { hits.hits.0._source.version: "1.1.12" }
|
||||
- match: { hits.hits.1._source.version: "2.0.0-beta" }
|
||||
|
||||
---
|
||||
"Sort script":
|
||||
- do:
|
||||
search:
|
||||
index: test_index
|
||||
body:
|
||||
sort: { "_script" : { "type" : "number", "script" : { "source": "doc['version'].value.length()" } } }
|
||||
|
||||
- match: { hits.total.value: 3 }
|
||||
- match: { hits.hits.0._source.version: "3.1.0" }
|
||||
- match: { hits.hits.1._source.version: "1.1.12" }
|
||||
- match: { hits.hits.2._source.version: "2.0.0-beta" }
|
||||
23
x-pack/plugin/versionfield/build.gradle
Normal file
23
x-pack/plugin/versionfield/build.gradle
Normal file
@ -0,0 +1,23 @@
|
||||
evaluationDependsOn(xpackModule('core'))
|
||||
|
||||
apply plugin: 'elasticsearch.esplugin'
|
||||
apply plugin: 'elasticsearch.internal-cluster-test'
|
||||
|
||||
esplugin {
|
||||
name 'versionfield'
|
||||
description 'A plugin for a field type to store sofware versions'
|
||||
classname 'org.elasticsearch.xpack.versionfield.VersionFieldPlugin'
|
||||
extendedPlugins = ['x-pack-core', 'lang-painless']
|
||||
}
|
||||
archivesBaseName = 'x-pack-versionfield'
|
||||
|
||||
dependencies {
|
||||
compileOnly project(path: xpackModule('core'), configuration: 'default')
|
||||
compileOnly project(':modules:lang-painless:spi')
|
||||
compileOnly(project(':modules:lang-painless')) {
|
||||
// exclude ASM to not affect featureAware task on Java 10+
|
||||
exclude group: "org.ow2.asm"
|
||||
}
|
||||
testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
|
||||
testImplementation project(path: xpackModule('analytics'), configuration: 'default')
|
||||
}
|
||||
@ -0,0 +1,89 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.versionfield;
|
||||
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.search.aggregations.AggregationBuilders;
|
||||
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
|
||||
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
|
||||
import org.elasticsearch.test.ESIntegTestCase;
|
||||
import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
||||
|
||||
public class VersionFieldIT extends ESIntegTestCase {
|
||||
|
||||
@Override
|
||||
protected Collection<Class<? extends Plugin>> nodePlugins() {
|
||||
return org.elasticsearch.common.collect.List.of(VersionFieldPlugin.class, LocalStateCompositeXPackPlugin.class);
|
||||
}
|
||||
|
||||
public void testTermsAggregation() throws Exception {
|
||||
String indexName = "test";
|
||||
createIndex(indexName);
|
||||
|
||||
client().admin()
|
||||
.indices()
|
||||
.preparePutMapping(indexName)
|
||||
.setType("_doc")
|
||||
.setSource(
|
||||
XContentFactory.jsonBuilder()
|
||||
.startObject()
|
||||
.startObject("_doc")
|
||||
.startObject("properties")
|
||||
.startObject("version")
|
||||
.field("type", "version")
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
)
|
||||
.get();
|
||||
ensureGreen();
|
||||
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("1")
|
||||
.setSource(jsonBuilder().startObject().field("version", "1.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("2")
|
||||
.setSource(jsonBuilder().startObject().field("version", "1.3.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("3")
|
||||
.setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("4")
|
||||
.setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("5")
|
||||
.setSource(jsonBuilder().startObject().field("version", "3.11.5").endObject())
|
||||
.get();
|
||||
refresh();
|
||||
|
||||
// terms aggs
|
||||
SearchResponse response = client().prepareSearch(indexName)
|
||||
.addAggregation(AggregationBuilders.terms("myterms").field("version"))
|
||||
.get();
|
||||
Terms terms = response.getAggregations().get("myterms");
|
||||
List<? extends Bucket> buckets = terms.getBuckets();
|
||||
|
||||
assertEquals(5, buckets.size());
|
||||
assertEquals("1.0", buckets.get(0).getKey());
|
||||
assertEquals("1.3.0", buckets.get(1).getKey());
|
||||
assertEquals("2.1.0-alpha", buckets.get(2).getKey());
|
||||
assertEquals("2.1.0", buckets.get(3).getKey());
|
||||
assertEquals("3.11.5", buckets.get(4).getKey());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,226 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.versionfield;
|
||||
|
||||
import org.apache.commons.codec.Charsets;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.BytesRefBuilder;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Encodes a version string to a {@link BytesRef} that correctly sorts according to software version precedence rules like
|
||||
* the ones described in Semantic Versioning (https://semver.org/)
|
||||
*
|
||||
* Version strings are considered to consist of three parts:
|
||||
* <ul>
|
||||
* <li> a numeric major.minor.patch part starting the version string (e.g. 1.2.3)
|
||||
* <li> an optional "pre-release" part that starts with a `-` character and can consist of several alphanumerical sections
|
||||
* separated by dots (e.g. "-alpha.2.3")
|
||||
* <li> an optional "build" part that starts with a `+` character. This will simply be treated as a suffix with ASCII sort order.
|
||||
* </ul>
|
||||
*
|
||||
* The version string is encoded such that the ordering works like the following:
|
||||
* <ul>
|
||||
* <li> Major, minor, and patch versions are always compared numerically
|
||||
* <li> pre-release version have lower precedence than a normal version. (e.g 1.0.0-alpha < 1.0.0)
|
||||
* <li> the precedence for pre-release versions with same main version is calculated comparing each dot separated identifier from
|
||||
* left to right. Identifiers consisting of only digits are compared numerically and identifiers with letters or hyphens are compared
|
||||
* lexically in ASCII sort order. Numeric identifiers always have lower precedence than non-numeric identifiers.
|
||||
* </ul>
|
||||
*/
|
||||
class VersionEncoder {
|
||||
|
||||
public static final byte NUMERIC_MARKER_BYTE = (byte) 0x01;
|
||||
public static final byte PRERELEASE_SEPARATOR_BYTE = (byte) 0x02;
|
||||
public static final byte NO_PRERELEASE_SEPARATOR_BYTE = (byte) 0x03;
|
||||
|
||||
private static final char PRERELEASE_SEPARATOR = '-';
|
||||
private static final char DOT_SEPARATOR = '.';
|
||||
private static final char BUILD_SEPARATOR = '+';
|
||||
private static final String ENCODED_EMPTY_STRING = new String(new byte[] { NO_PRERELEASE_SEPARATOR_BYTE }, Charsets.UTF_8);
|
||||
|
||||
// Regex to test relaxed Semver Main Version validity. Allows for more or less than three main version parts
|
||||
private static Pattern LEGAL_MAIN_VERSION_SEMVER = Pattern.compile("(0|[1-9]\\d*)(\\.(0|[1-9]\\d*))*");
|
||||
|
||||
private static Pattern LEGAL_PRERELEASE_VERSION_SEMVER = Pattern.compile(
|
||||
"(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))"
|
||||
);
|
||||
|
||||
private static Pattern LEGAL_BUILDSUFFIX_SEMVER = Pattern.compile("(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?");
|
||||
|
||||
/**
|
||||
* Encodes a version string.
|
||||
*/
|
||||
public static EncodedVersion encodeVersion(String versionString) {
|
||||
VersionParts versionParts = VersionParts.ofVersion(versionString);
|
||||
|
||||
// don't treat non-legal versions further, just mark them as illegal and return
|
||||
if (legalVersionString(versionParts) == false) {
|
||||
if (versionString.length() == 0) {
|
||||
// special case, we want empty string to sort after valid strings, which all start with 0x01, add a higher char that
|
||||
// we are sure to remove when decoding
|
||||
versionString = ENCODED_EMPTY_STRING;
|
||||
}
|
||||
return new EncodedVersion(new BytesRef(versionString), false, true, null, null, null);
|
||||
}
|
||||
|
||||
BytesRefBuilder encodedBytes = new BytesRefBuilder();
|
||||
Integer[] mainVersionParts = prefixDigitGroupsWithLength(versionParts.mainVersion, encodedBytes);
|
||||
|
||||
if (versionParts.preRelease != null) {
|
||||
encodedBytes.append(PRERELEASE_SEPARATOR_BYTE); // versions with pre-release part sort before ones without
|
||||
encodedBytes.append((byte) PRERELEASE_SEPARATOR);
|
||||
String[] preReleaseParts = versionParts.preRelease.substring(1).split("\\.");
|
||||
boolean first = true;
|
||||
for (String preReleasePart : preReleaseParts) {
|
||||
if (first == false) {
|
||||
encodedBytes.append((byte) DOT_SEPARATOR);
|
||||
}
|
||||
boolean isNumeric = preReleasePart.chars().allMatch(x -> Character.isDigit(x));
|
||||
if (isNumeric) {
|
||||
prefixDigitGroupsWithLength(preReleasePart, encodedBytes);
|
||||
} else {
|
||||
encodedBytes.append(new BytesRef(preReleasePart));
|
||||
}
|
||||
first = false;
|
||||
}
|
||||
} else {
|
||||
encodedBytes.append(NO_PRERELEASE_SEPARATOR_BYTE);
|
||||
}
|
||||
|
||||
if (versionParts.buildSuffix != null) {
|
||||
encodedBytes.append(new BytesRef(versionParts.buildSuffix));
|
||||
}
|
||||
return new EncodedVersion(
|
||||
encodedBytes.toBytesRef(),
|
||||
true,
|
||||
versionParts.preRelease != null,
|
||||
mainVersionParts[0],
|
||||
mainVersionParts[1],
|
||||
mainVersionParts[2]
|
||||
);
|
||||
}
|
||||
|
||||
private static Integer[] prefixDigitGroupsWithLength(String input, BytesRefBuilder result) {
|
||||
int pos = 0;
|
||||
int mainVersionCounter = 0;
|
||||
Integer[] mainVersionComponents = new Integer[3];
|
||||
while (pos < input.length()) {
|
||||
if (Character.isDigit(input.charAt(pos))) {
|
||||
// found beginning of number block, so get its length
|
||||
int start = pos;
|
||||
BytesRefBuilder number = new BytesRefBuilder();
|
||||
while (pos < input.length() && Character.isDigit(input.charAt(pos))) {
|
||||
number.append((byte) input.charAt(pos));
|
||||
pos++;
|
||||
}
|
||||
int length = pos - start;
|
||||
if (length >= 128) {
|
||||
throw new IllegalArgumentException("Groups of digits cannot be longer than 127, but found: " + length);
|
||||
}
|
||||
result.append(NUMERIC_MARKER_BYTE); // ensure length byte does cause higher sort order comparing to other byte[]
|
||||
result.append((byte) (length | 0x80)); // add upper bit to mark as length
|
||||
result.append(number);
|
||||
|
||||
// if present, parse out three leftmost version parts
|
||||
if (mainVersionCounter < 3) {
|
||||
mainVersionComponents[mainVersionCounter] = Integer.valueOf(number.toBytesRef().utf8ToString());
|
||||
mainVersionCounter++;
|
||||
}
|
||||
} else {
|
||||
result.append((byte) input.charAt(pos));
|
||||
pos++;
|
||||
}
|
||||
}
|
||||
return mainVersionComponents;
|
||||
}
|
||||
|
||||
public static String decodeVersion(BytesRef version) {
|
||||
int inputPos = version.offset;
|
||||
int resultPos = 0;
|
||||
byte[] result = new byte[version.length];
|
||||
while (inputPos < version.offset + version.length) {
|
||||
byte inputByte = version.bytes[inputPos];
|
||||
if (inputByte == NUMERIC_MARKER_BYTE) {
|
||||
// need to skip this byte
|
||||
inputPos++;
|
||||
// this should always be a length encoding, which is skipped by increasing inputPos at the end of the loop
|
||||
assert version.bytes[inputPos] < 0;
|
||||
} else if (inputByte != PRERELEASE_SEPARATOR_BYTE && inputByte != NO_PRERELEASE_SEPARATOR_BYTE) {
|
||||
result[resultPos] = inputByte;
|
||||
resultPos++;
|
||||
}
|
||||
inputPos++;
|
||||
}
|
||||
return new String(result, 0, resultPos, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
static boolean legalVersionString(VersionParts versionParts) {
|
||||
boolean legalMainVersion = LEGAL_MAIN_VERSION_SEMVER.matcher(versionParts.mainVersion).matches();
|
||||
boolean legalPreRelease = true;
|
||||
if (versionParts.preRelease != null) {
|
||||
legalPreRelease = LEGAL_PRERELEASE_VERSION_SEMVER.matcher(versionParts.preRelease).matches();
|
||||
}
|
||||
boolean legalBuildSuffix = true;
|
||||
if (versionParts.buildSuffix != null) {
|
||||
legalBuildSuffix = LEGAL_BUILDSUFFIX_SEMVER.matcher(versionParts.buildSuffix).matches();
|
||||
}
|
||||
return legalMainVersion && legalPreRelease && legalBuildSuffix;
|
||||
}
|
||||
|
||||
static class EncodedVersion {
|
||||
|
||||
public final boolean isLegal;
|
||||
public final boolean isPreRelease;
|
||||
public final BytesRef bytesRef;
|
||||
public final Integer major;
|
||||
public final Integer minor;
|
||||
public final Integer patch;
|
||||
|
||||
private EncodedVersion(BytesRef bytesRef, boolean isLegal, boolean isPreRelease, Integer major, Integer minor, Integer patch) {
|
||||
super();
|
||||
this.bytesRef = bytesRef;
|
||||
this.isLegal = isLegal;
|
||||
this.isPreRelease = isPreRelease;
|
||||
this.major = major;
|
||||
this.minor = minor;
|
||||
this.patch = patch;
|
||||
}
|
||||
}
|
||||
|
||||
static class VersionParts {
|
||||
final String mainVersion;
|
||||
final String preRelease;
|
||||
final String buildSuffix;
|
||||
|
||||
private VersionParts(String mainVersion, String preRelease, String buildSuffix) {
|
||||
this.mainVersion = mainVersion;
|
||||
this.preRelease = preRelease;
|
||||
this.buildSuffix = buildSuffix;
|
||||
}
|
||||
|
||||
static VersionParts ofVersion(String versionString) {
|
||||
String buildSuffix = extractSuffix(versionString, BUILD_SEPARATOR);
|
||||
if (buildSuffix != null) {
|
||||
versionString = versionString.substring(0, versionString.length() - buildSuffix.length());
|
||||
}
|
||||
|
||||
String preRelease = extractSuffix(versionString, PRERELEASE_SEPARATOR);
|
||||
if (preRelease != null) {
|
||||
versionString = versionString.substring(0, versionString.length() - preRelease.length());
|
||||
}
|
||||
return new VersionParts(versionString, preRelease, buildSuffix);
|
||||
}
|
||||
|
||||
private static String extractSuffix(String input, char separator) {
|
||||
int start = input.indexOf(separator);
|
||||
return start > 0 ? input.substring(start) : null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.versionfield;
|
||||
|
||||
import org.elasticsearch.painless.spi.PainlessExtension;
|
||||
import org.elasticsearch.painless.spi.Whitelist;
|
||||
import org.elasticsearch.painless.spi.WhitelistLoader;
|
||||
import org.elasticsearch.script.AggregationScript;
|
||||
import org.elasticsearch.script.FieldScript;
|
||||
import org.elasticsearch.script.FilterScript;
|
||||
import org.elasticsearch.script.NumberSortScript;
|
||||
import org.elasticsearch.script.ScoreScript;
|
||||
import org.elasticsearch.script.ScriptContext;
|
||||
import org.elasticsearch.script.StringSortScript;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static java.util.Collections.singletonList;
|
||||
|
||||
public class VersionFieldDocValuesExtension implements PainlessExtension {
|
||||
|
||||
private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles(VersionFieldDocValuesExtension.class, "whitelist.txt");
|
||||
|
||||
@Override
|
||||
public Map<ScriptContext<?>, List<Whitelist>> getContextWhitelists() {
|
||||
Map<ScriptContext<?>, List<Whitelist>> whitelist = new HashMap<>();
|
||||
List<Whitelist> list = singletonList(WHITELIST);
|
||||
whitelist.put(AggregationScript.CONTEXT, list);
|
||||
whitelist.put(ScoreScript.CONTEXT, list);
|
||||
whitelist.put(FilterScript.CONTEXT, list);
|
||||
whitelist.put(FieldScript.CONTEXT, list);
|
||||
whitelist.put(NumberSortScript.CONTEXT, list);
|
||||
whitelist.put(StringSortScript.CONTEXT, list);
|
||||
return whitelist;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.versionfield;
|
||||
|
||||
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.mapper.Mapper;
|
||||
import org.elasticsearch.plugins.MapperPlugin;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.search.DocValueFormat;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class VersionFieldPlugin extends Plugin implements MapperPlugin {
|
||||
|
||||
public VersionFieldPlugin(Settings settings) {}
|
||||
|
||||
@Override
|
||||
public Map<String, Mapper.TypeParser> getMappers() {
|
||||
return org.elasticsearch.common.collect.Map.of(VersionStringFieldMapper.CONTENT_TYPE, VersionStringFieldMapper.PARSER);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
|
||||
return org.elasticsearch.common.collect.List.of(
|
||||
new NamedWriteableRegistry.Entry(
|
||||
DocValueFormat.class,
|
||||
VersionStringFieldMapper.VERSION_DOCVALUE.getWriteableName(),
|
||||
in -> VersionStringFieldMapper.VERSION_DOCVALUE
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,120 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.versionfield;
|
||||
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.AutomatonQuery;
|
||||
import org.apache.lucene.search.WildcardQuery;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.automaton.Automata;
|
||||
import org.apache.lucene.util.automaton.Automaton;
|
||||
import org.apache.lucene.util.automaton.Operations;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* A variation of the {@link WildcardQuery} than skips over meta characters introduced using {@link VersionEncoder}.
|
||||
*/
|
||||
class VersionFieldWildcardQuery extends AutomatonQuery {
|
||||
|
||||
private static final Automaton OPTIONAL_NUMERIC_CHARPREFIX = Operations.optional(
|
||||
Operations.concatenate(Automata.makeChar(VersionEncoder.NUMERIC_MARKER_BYTE), Automata.makeCharRange(0x80, 0xFF))
|
||||
);
|
||||
|
||||
private static final Automaton OPTIONAL_RELEASE_SEPARATOR = Operations.optional(
|
||||
Operations.union(
|
||||
Automata.makeChar(VersionEncoder.PRERELEASE_SEPARATOR_BYTE),
|
||||
Automata.makeChar(VersionEncoder.NO_PRERELEASE_SEPARATOR_BYTE)
|
||||
)
|
||||
);
|
||||
|
||||
private static final byte WILDCARD_STRING = '*';
|
||||
|
||||
private static final byte WILDCARD_CHAR = '?';
|
||||
|
||||
VersionFieldWildcardQuery(Term term) {
|
||||
super(term, toAutomaton(term), Integer.MAX_VALUE, true);
|
||||
}
|
||||
|
||||
private static Automaton toAutomaton(Term wildcardquery) {
|
||||
List<Automaton> automata = new ArrayList<>();
|
||||
|
||||
BytesRef wildcardText = wildcardquery.bytes();
|
||||
boolean containsPreReleaseSeparator = false;
|
||||
|
||||
for (int i = 0; i < wildcardText.length;) {
|
||||
final byte c = wildcardText.bytes[wildcardText.offset + i];
|
||||
int length = Character.charCount(c);
|
||||
|
||||
switch (c) {
|
||||
case WILDCARD_STRING:
|
||||
automata.add(Automata.makeAnyString());
|
||||
break;
|
||||
case WILDCARD_CHAR:
|
||||
// this should also match leading digits, which have optional leading numeric marker and length bytes
|
||||
automata.add(OPTIONAL_NUMERIC_CHARPREFIX);
|
||||
automata.add(OPTIONAL_RELEASE_SEPARATOR);
|
||||
automata.add(Automata.makeAnyChar());
|
||||
break;
|
||||
|
||||
case '-':
|
||||
// this should potentially match the first prerelease-dash, so we need an optional marker byte here
|
||||
automata.add(Operations.optional(Automata.makeChar(VersionEncoder.PRERELEASE_SEPARATOR_BYTE)));
|
||||
containsPreReleaseSeparator = true;
|
||||
automata.add(Automata.makeChar(c));
|
||||
break;
|
||||
case '+':
|
||||
// this can potentially appear after major version, optionally match the no-prerelease marker
|
||||
automata.add(Operations.optional(Automata.makeChar(VersionEncoder.NO_PRERELEASE_SEPARATOR_BYTE)));
|
||||
containsPreReleaseSeparator = true;
|
||||
automata.add(Automata.makeChar(c));
|
||||
break;
|
||||
case '0':
|
||||
case '1':
|
||||
case '2':
|
||||
case '3':
|
||||
case '4':
|
||||
case '5':
|
||||
case '6':
|
||||
case '7':
|
||||
case '8':
|
||||
case '9':
|
||||
boolean firstDigitInGroup = true;
|
||||
if (i > 0
|
||||
&& wildcardText.bytes[wildcardText.offset + i - 1] >= (byte) '0'
|
||||
&& wildcardText.bytes[wildcardText.offset + i - 1] <= (byte) '9') {
|
||||
firstDigitInGroup = false;
|
||||
}
|
||||
if (firstDigitInGroup) {
|
||||
automata.add(OPTIONAL_NUMERIC_CHARPREFIX);
|
||||
}
|
||||
automata.add(Automata.makeChar(c));
|
||||
break;
|
||||
default:
|
||||
automata.add(Automata.makeChar(c));
|
||||
}
|
||||
i += length;
|
||||
}
|
||||
// when we only have main version part, we need to add an optional NO_PRERELESE_SEPARATOR_BYTE
|
||||
if (containsPreReleaseSeparator == false) {
|
||||
automata.add(Operations.optional(Automata.makeChar(VersionEncoder.NO_PRERELEASE_SEPARATOR_BYTE)));
|
||||
}
|
||||
return Operations.concatenate(automata);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString(String field) {
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
if (!getField().equals(field)) {
|
||||
buffer.append(getField());
|
||||
buffer.append(":");
|
||||
}
|
||||
buffer.append(term.text());
|
||||
return buffer.toString();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,58 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.versionfield;
|
||||
|
||||
import org.apache.lucene.index.SortedSetDocValues;
|
||||
import org.apache.lucene.util.ArrayUtil;
|
||||
import org.elasticsearch.index.fielddata.ScriptDocValues;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public final class VersionScriptDocValues extends ScriptDocValues<String> {
|
||||
|
||||
private final SortedSetDocValues in;
|
||||
private long[] ords = new long[0];
|
||||
private int count;
|
||||
|
||||
public VersionScriptDocValues(SortedSetDocValues in) {
|
||||
this.in = in;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setNextDocId(int docId) throws IOException {
|
||||
count = 0;
|
||||
if (in.advanceExact(docId)) {
|
||||
for (long ord = in.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = in.nextOrd()) {
|
||||
ords = ArrayUtil.grow(ords, count + 1);
|
||||
ords[count++] = ord;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public String getValue() {
|
||||
return get(0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String get(int index) {
|
||||
if (count == 0) {
|
||||
throw new IllegalStateException(
|
||||
"A document doesn't have a value for a field! " + "Use doc[<field>].size()==0 to check if a document is missing a field!"
|
||||
);
|
||||
}
|
||||
try {
|
||||
return VersionEncoder.decodeVersion(in.lookupOrd(ords[index]));
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int size() {
|
||||
return count;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,404 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.versionfield;
|
||||
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.document.FieldType;
|
||||
import org.apache.lucene.document.SortedSetDocValuesField;
|
||||
import org.apache.lucene.index.FilteredTermsEnum;
|
||||
import org.apache.lucene.index.IndexOptions;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.index.Terms;
|
||||
import org.apache.lucene.index.TermsEnum;
|
||||
import org.apache.lucene.search.DocValuesFieldExistsQuery;
|
||||
import org.apache.lucene.search.FuzzyQuery;
|
||||
import org.apache.lucene.search.MultiTermQuery;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.RegexpQuery;
|
||||
import org.apache.lucene.search.TermRangeQuery;
|
||||
import org.apache.lucene.util.AttributeSource;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.automaton.ByteRunAutomaton;
|
||||
import org.elasticsearch.ElasticsearchException;
|
||||
import org.elasticsearch.common.Nullable;
|
||||
import org.elasticsearch.common.collect.Iterators;
|
||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||
import org.elasticsearch.common.lucene.BytesRefs;
|
||||
import org.elasticsearch.common.lucene.Lucene;
|
||||
import org.elasticsearch.common.unit.Fuzziness;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.mapper.MappedFieldType;
|
||||
import org.elasticsearch.index.mapper.Mapper;
|
||||
import org.elasticsearch.index.mapper.MapperService;
|
||||
import org.elasticsearch.index.mapper.ParametrizedFieldMapper;
|
||||
import org.elasticsearch.index.mapper.ParseContext;
|
||||
import org.elasticsearch.index.mapper.SourceValueFetcher;
|
||||
import org.elasticsearch.index.mapper.TermBasedFieldType;
|
||||
import org.elasticsearch.index.mapper.TextSearchInfo;
|
||||
import org.elasticsearch.index.mapper.ValueFetcher;
|
||||
import org.elasticsearch.index.query.QueryShardContext;
|
||||
import org.elasticsearch.index.query.support.QueryParsers;
|
||||
import org.elasticsearch.search.DocValueFormat;
|
||||
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
|
||||
import org.elasticsearch.search.lookup.SearchLookup;
|
||||
import org.elasticsearch.xpack.versionfield.VersionEncoder.EncodedVersion;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.ZoneId;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
|
||||
import static org.elasticsearch.xpack.versionfield.VersionEncoder.encodeVersion;
|
||||
|
||||
/**
|
||||
* A {@link FieldMapper} for indexing fields with version strings.
|
||||
*/
|
||||
public class VersionStringFieldMapper extends ParametrizedFieldMapper {
|
||||
|
||||
private static byte[] MIN_VALUE = new byte[16];
|
||||
private static byte[] MAX_VALUE = new byte[16];
|
||||
static {
|
||||
Arrays.fill(MIN_VALUE, (byte) 0);
|
||||
Arrays.fill(MAX_VALUE, (byte) -1);
|
||||
}
|
||||
|
||||
public static final String CONTENT_TYPE = "version";
|
||||
|
||||
public static class Defaults {
|
||||
public static final FieldType FIELD_TYPE = new FieldType();
|
||||
|
||||
static {
|
||||
FIELD_TYPE.setTokenized(false);
|
||||
FIELD_TYPE.setOmitNorms(true);
|
||||
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
|
||||
FIELD_TYPE.freeze();
|
||||
}
|
||||
}
|
||||
|
||||
static class Builder extends ParametrizedFieldMapper.Builder {
|
||||
|
||||
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
|
||||
|
||||
Builder(String name) {
|
||||
super(name);
|
||||
}
|
||||
|
||||
private VersionStringFieldType buildFieldType(BuilderContext context, FieldType fieldtype) {
|
||||
return new VersionStringFieldType(buildFullName(context), fieldtype, meta.getValue());
|
||||
}
|
||||
|
||||
@Override
|
||||
public VersionStringFieldMapper build(BuilderContext context) {
|
||||
FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE);
|
||||
return new VersionStringFieldMapper(
|
||||
name,
|
||||
fieldtype,
|
||||
buildFieldType(context, fieldtype),
|
||||
multiFieldsBuilder.build(this, context),
|
||||
copyTo.build()
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<Parameter<?>> getParameters() {
|
||||
return org.elasticsearch.common.collect.List.of(meta);
|
||||
}
|
||||
}
|
||||
|
||||
public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));
|
||||
|
||||
public static final class VersionStringFieldType extends TermBasedFieldType {
|
||||
|
||||
public VersionStringFieldType(String name, FieldType fieldType, Map<String, String> meta) {
|
||||
super(name, true, true, new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), meta);
|
||||
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String typeName() {
|
||||
return CONTENT_TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Query existsQuery(QueryShardContext context) {
|
||||
return new DocValuesFieldExistsQuery(name());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
|
||||
return wildcardQuery(value + "*", method, context);
|
||||
}
|
||||
|
||||
/**
|
||||
* We cannot simply use RegexpQuery directly since we use the encoded terms from the dictionary, but the
|
||||
* automaton in the query will assume unencoded terms. We are running through all terms, decode them and
|
||||
* then run them through the automaton manually instead. This is not as efficient as intersecting the original
|
||||
* Terms with the compiled automaton, but we expect the number of distinct version terms indexed into this field
|
||||
* to be low enough and the use of "regexp" queries on this field rare enough to brute-force this
|
||||
*/
|
||||
@Override
|
||||
public Query regexpQuery(
|
||||
String value,
|
||||
int syntaxFlags,
|
||||
int matchFlags,
|
||||
int maxDeterminizedStates,
|
||||
@Nullable MultiTermQuery.RewriteMethod method,
|
||||
QueryShardContext context
|
||||
) {
|
||||
if (context.allowExpensiveQueries() == false) {
|
||||
throw new ElasticsearchException(
|
||||
"[regexp] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false."
|
||||
);
|
||||
}
|
||||
RegexpQuery query = new RegexpQuery(new Term(name(), new BytesRef(value)), syntaxFlags, matchFlags, maxDeterminizedStates) {
|
||||
|
||||
@Override
|
||||
protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException {
|
||||
return new FilteredTermsEnum(terms.iterator(), false) {
|
||||
|
||||
@Override
|
||||
protected AcceptStatus accept(BytesRef term) throws IOException {
|
||||
byte[] decoded = VersionEncoder.decodeVersion(term).getBytes(StandardCharsets.UTF_8);
|
||||
boolean accepted = compiled.runAutomaton.run(decoded, 0, decoded.length);
|
||||
if (accepted) {
|
||||
return AcceptStatus.YES;
|
||||
}
|
||||
return AcceptStatus.NO;
|
||||
}
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
if (method != null) {
|
||||
query.setRewriteMethod(method);
|
||||
}
|
||||
return query;
|
||||
}
|
||||
|
||||
/**
|
||||
* We cannot simply use FuzzyQuery directly since we use the encoded terms from the dictionary, but the
|
||||
* automaton in the query will assume unencoded terms. We are running through all terms, decode them and
|
||||
* then run them through the automaton manually instead. This is not as efficient as intersecting the original
|
||||
* Terms with the compiled automaton, but we expect the number of distinct version terms indexed into this field
|
||||
* to be low enough and the use of "fuzzy" queries on this field rare enough to brute-force this
|
||||
*/
|
||||
@Override
|
||||
public Query fuzzyQuery(
|
||||
Object value,
|
||||
Fuzziness fuzziness,
|
||||
int prefixLength,
|
||||
int maxExpansions,
|
||||
boolean transpositions,
|
||||
QueryShardContext context
|
||||
) {
|
||||
if (context.allowExpensiveQueries() == false) {
|
||||
throw new ElasticsearchException(
|
||||
"[fuzzy] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false."
|
||||
);
|
||||
}
|
||||
return new FuzzyQuery(
|
||||
new Term(name(), (BytesRef) value),
|
||||
fuzziness.asDistance(BytesRefs.toString(value)),
|
||||
prefixLength,
|
||||
maxExpansions,
|
||||
transpositions
|
||||
) {
|
||||
@Override
|
||||
protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException {
|
||||
ByteRunAutomaton runAutomaton = getAutomata().runAutomaton;
|
||||
|
||||
return new FilteredTermsEnum(terms.iterator(), false) {
|
||||
|
||||
@Override
|
||||
protected AcceptStatus accept(BytesRef term) throws IOException {
|
||||
byte[] decoded = VersionEncoder.decodeVersion(term).getBytes(StandardCharsets.UTF_8);
|
||||
boolean accepted = runAutomaton.run(decoded, 0, decoded.length);
|
||||
if (accepted) {
|
||||
return AcceptStatus.YES;
|
||||
}
|
||||
return AcceptStatus.NO;
|
||||
}
|
||||
};
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
|
||||
if (context.allowExpensiveQueries() == false) {
|
||||
throw new ElasticsearchException(
|
||||
"[wildcard] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false."
|
||||
);
|
||||
}
|
||||
|
||||
VersionFieldWildcardQuery query = new VersionFieldWildcardQuery(new Term(name(), value));
|
||||
QueryParsers.setRewriteMethod(query, method);
|
||||
return query;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected BytesRef indexedValueForSearch(Object value) {
|
||||
String valueAsString;
|
||||
if (value instanceof String) {
|
||||
valueAsString = (String) value;
|
||||
} else if (value instanceof BytesRef) {
|
||||
// encoded string, need to re-encode
|
||||
valueAsString = ((BytesRef) value).utf8ToString();
|
||||
} else {
|
||||
throw new IllegalArgumentException("Illegal value type: " + value.getClass() + ", value: " + value);
|
||||
}
|
||||
return encodeVersion(valueAsString).bytesRef;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
|
||||
return new SortedSetOrdinalsIndexFieldData.Builder(name(), VersionScriptDocValues::new, CoreValuesSourceType.BYTES);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object valueForDisplay(Object value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
return VERSION_DOCVALUE.format((BytesRef) value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
|
||||
if (format != null) {
|
||||
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support custom formats");
|
||||
}
|
||||
if (timeZone != null) {
|
||||
throw new IllegalArgumentException(
|
||||
"Field [" + name() + "] of type [" + typeName() + "] does not support custom time zones"
|
||||
);
|
||||
}
|
||||
return VERSION_DOCVALUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
|
||||
BytesRef lower = lowerTerm == null ? null : indexedValueForSearch(lowerTerm);
|
||||
BytesRef upper = upperTerm == null ? null : indexedValueForSearch(upperTerm);
|
||||
return new TermRangeQuery(name(), lower, upper, includeLower, includeUpper);
|
||||
}
|
||||
}
|
||||
|
||||
private final FieldType fieldType;
|
||||
|
||||
private VersionStringFieldMapper(
|
||||
String simpleName,
|
||||
FieldType fieldType,
|
||||
MappedFieldType mappedFieldType,
|
||||
MultiFields multiFields,
|
||||
CopyTo copyTo
|
||||
) {
|
||||
super(simpleName, mappedFieldType, multiFields, copyTo);
|
||||
this.fieldType = fieldType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public VersionStringFieldType fieldType() {
|
||||
return (VersionStringFieldType) super.fieldType();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
|
||||
if (format != null) {
|
||||
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
|
||||
}
|
||||
|
||||
return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), null) {
|
||||
@Override
|
||||
protected String parseSourceValue(Object value) {
|
||||
return value.toString();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String contentType() {
|
||||
return CONTENT_TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected VersionStringFieldMapper clone() {
|
||||
return (VersionStringFieldMapper) super.clone();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void parseCreateField(ParseContext context) throws IOException {
|
||||
String versionString;
|
||||
if (context.externalValueSet()) {
|
||||
versionString = context.externalValue().toString();
|
||||
} else {
|
||||
XContentParser parser = context.parser();
|
||||
if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
|
||||
return;
|
||||
} else {
|
||||
versionString = parser.textOrNull();
|
||||
}
|
||||
}
|
||||
|
||||
if (versionString == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
EncodedVersion encoding = encodeVersion(versionString);
|
||||
BytesRef encodedVersion = encoding.bytesRef;
|
||||
context.doc().add(new Field(fieldType().name(), encodedVersion, fieldType));
|
||||
context.doc().add(new SortedSetDocValuesField(fieldType().name(), encodedVersion));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<Mapper> iterator() {
|
||||
List<Mapper> subIterators = new ArrayList<>();
|
||||
@SuppressWarnings("unchecked")
|
||||
Iterator<Mapper> concat = Iterators.concat(super.iterator(), subIterators.iterator());
|
||||
return concat;
|
||||
}
|
||||
|
||||
public static DocValueFormat VERSION_DOCVALUE = new DocValueFormat() {
|
||||
|
||||
@Override
|
||||
public String getWriteableName() {
|
||||
return "version";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeTo(StreamOutput out) {}
|
||||
|
||||
@Override
|
||||
public String format(BytesRef value) {
|
||||
return VersionEncoder.decodeVersion(value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef parseBytesRef(String value) {
|
||||
return VersionEncoder.encodeVersion(value).bytesRef;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getWriteableName();
|
||||
}
|
||||
};
|
||||
|
||||
@Override
|
||||
public ParametrizedFieldMapper.Builder getMergeBuilder() {
|
||||
return new Builder(simpleName()).init(this);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
org.elasticsearch.xpack.versionfield.VersionFieldDocValuesExtension
|
||||
@ -0,0 +1,5 @@
|
||||
|
||||
class org.elasticsearch.xpack.versionfield.VersionScriptDocValues {
|
||||
String get(int)
|
||||
String getValue()
|
||||
}
|
||||
@ -0,0 +1,237 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.versionfield;
|
||||
|
||||
import joptsimple.internal.Strings;
|
||||
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
import org.elasticsearch.xpack.versionfield.VersionEncoder.EncodedVersion;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
import static org.elasticsearch.xpack.versionfield.VersionEncoder.decodeVersion;
|
||||
|
||||
public class VersionEncoderTests extends ESTestCase {
|
||||
|
||||
public void testEncodingOrderingSemver() {
|
||||
assertTrue(encodeVersion("1").compareTo(encodeVersion("1.0")) < 0);
|
||||
assertTrue(encodeVersion("1.0").compareTo(encodeVersion("1.0.0.0.0.0.0.0.0.1")) < 0);
|
||||
assertTrue(encodeVersion("1.0.0").compareTo(encodeVersion("1.0.0.0.0.0.0.0.0.1")) < 0);
|
||||
assertTrue(encodeVersion("1.0.0").compareTo(encodeVersion("2.0.0")) < 0);
|
||||
assertTrue(encodeVersion("2.0.0").compareTo(encodeVersion("11.0.0")) < 0);
|
||||
assertTrue(encodeVersion("2.0.0").compareTo(encodeVersion("2.1.0")) < 0);
|
||||
assertTrue(encodeVersion("2.1.0").compareTo(encodeVersion("2.1.1")) < 0);
|
||||
assertTrue(encodeVersion("2.1.1").compareTo(encodeVersion("2.1.1.0")) < 0);
|
||||
assertTrue(encodeVersion("2.0.0").compareTo(encodeVersion("11.0.0")) < 0);
|
||||
assertTrue(encodeVersion("1.0.0").compareTo(encodeVersion("2.0")) < 0);
|
||||
assertTrue(encodeVersion("1.0.0-a").compareTo(encodeVersion("1.0.0-b")) < 0);
|
||||
assertTrue(encodeVersion("1.0.0-1.0.0").compareTo(encodeVersion("1.0.0-2.0")) < 0);
|
||||
assertTrue(encodeVersion("1.0.0-alpha").compareTo(encodeVersion("1.0.0-alpha.1")) < 0);
|
||||
assertTrue(encodeVersion("1.0.0-alpha.1").compareTo(encodeVersion("1.0.0-alpha.beta")) < 0);
|
||||
assertTrue(encodeVersion("1.0.0-alpha.beta").compareTo(encodeVersion("1.0.0-beta")) < 0);
|
||||
assertTrue(encodeVersion("1.0.0-beta").compareTo(encodeVersion("1.0.0-beta.2")) < 0);
|
||||
assertTrue(encodeVersion("1.0.0-beta.2").compareTo(encodeVersion("1.0.0-beta.11")) < 0);
|
||||
assertTrue(encodeVersion("1.0.0-beta11").compareTo(encodeVersion("1.0.0-beta2")) < 0); // correct according to Semver specs
|
||||
assertTrue(encodeVersion("1.0.0-beta.11").compareTo(encodeVersion("1.0.0-rc.1")) < 0);
|
||||
assertTrue(encodeVersion("1.0.0-rc.1").compareTo(encodeVersion("1.0.0")) < 0);
|
||||
assertTrue(encodeVersion("1.0.0").compareTo(encodeVersion("2.0.0-pre127")) < 0);
|
||||
assertTrue(encodeVersion("2.0.0-pre127").compareTo(encodeVersion("2.0.0-pre128")) < 0);
|
||||
assertTrue(encodeVersion("2.0.0-pre128").compareTo(encodeVersion("2.0.0-pre128-somethingelse")) < 0);
|
||||
assertTrue(encodeVersion("2.0.0-pre20201231z110026").compareTo(encodeVersion("2.0.0-pre227")) < 0);
|
||||
// invalid versions sort after valid ones
|
||||
assertTrue(encodeVersion("99999.99999.99999").compareTo(encodeVersion("1.invalid")) < 0);
|
||||
assertTrue(encodeVersion("").compareTo(encodeVersion("a")) < 0);
|
||||
}
|
||||
|
||||
private static BytesRef encodeVersion(String version) {
|
||||
return VersionEncoder.encodeVersion(version).bytesRef;
|
||||
}
|
||||
|
||||
public void testPreReleaseFlag() {
|
||||
assertTrue(VersionEncoder.encodeVersion("1.2-alpha.beta").isPreRelease);
|
||||
assertTrue(VersionEncoder.encodeVersion("1.2.3-someOtherPreRelease").isPreRelease);
|
||||
assertTrue(VersionEncoder.encodeVersion("1.2.3-some-Other-Pre.123").isPreRelease);
|
||||
assertTrue(VersionEncoder.encodeVersion("1.2.3-some-Other-Pre.123+withBuild").isPreRelease);
|
||||
|
||||
assertFalse(VersionEncoder.encodeVersion("1").isPreRelease);
|
||||
assertFalse(VersionEncoder.encodeVersion("1.2").isPreRelease);
|
||||
assertFalse(VersionEncoder.encodeVersion("1.2.3").isPreRelease);
|
||||
assertFalse(VersionEncoder.encodeVersion("1.2.3+buildSufix").isPreRelease);
|
||||
assertFalse(VersionEncoder.encodeVersion("1.2.3+buildSufix-withDash").isPreRelease);
|
||||
}
|
||||
|
||||
public void testVersionPartExtraction() {
|
||||
int numParts = randomIntBetween(1, 6);
|
||||
String[] parts = new String[numParts];
|
||||
for (int i = 0; i < numParts; i++) {
|
||||
parts[i] = String.valueOf(randomIntBetween(1, 1000));
|
||||
}
|
||||
EncodedVersion encodedVersion = VersionEncoder.encodeVersion(String.join(".", parts));
|
||||
assertEquals(parts[0], encodedVersion.major.toString());
|
||||
if (numParts > 1) {
|
||||
assertEquals(parts[1], encodedVersion.minor.toString());
|
||||
} else {
|
||||
assertNull(encodedVersion.minor);
|
||||
}
|
||||
if (numParts > 2) {
|
||||
assertEquals(parts[2], encodedVersion.patch.toString());
|
||||
} else {
|
||||
assertNull(encodedVersion.patch);
|
||||
}
|
||||
}
|
||||
|
||||
public void testMaxDigitGroupLength() {
|
||||
|
||||
String versionString = "1.0." + Strings.repeat('1', 128);
|
||||
IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> decodeVersion(encodeVersion(versionString)));
|
||||
assertEquals("Groups of digits cannot be longer than 127, but found: 128", ex.getMessage());
|
||||
}
|
||||
|
||||
/**
|
||||
* test that encoding and decoding leads back to the same version string
|
||||
*/
|
||||
public void testRandomRoundtrip() {
|
||||
String versionString = randomVersionString();
|
||||
assertEquals(versionString, decodeVersion(encodeVersion(versionString)));
|
||||
}
|
||||
|
||||
private String randomVersionString() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(randomIntBetween(0, 1000));
|
||||
int releaseNumerals = randomIntBetween(0, 4);
|
||||
for (int i = 0; i < releaseNumerals; i++) {
|
||||
sb.append(".");
|
||||
sb.append(randomIntBetween(0, 10000));
|
||||
}
|
||||
// optional pre-release part
|
||||
if (randomBoolean()) {
|
||||
sb.append("-");
|
||||
int preReleaseParts = randomIntBetween(1, 5);
|
||||
for (int i = 0; i < preReleaseParts; i++) {
|
||||
if (randomBoolean()) {
|
||||
sb.append(randomIntBetween(0, 1000));
|
||||
} else {
|
||||
int alphanumParts = 3;
|
||||
for (int j = 0; j < alphanumParts; j++) {
|
||||
if (randomBoolean()) {
|
||||
sb.append(randomAlphaOfLengthBetween(1, 2));
|
||||
} else {
|
||||
sb.append(randomIntBetween(1, 99));
|
||||
}
|
||||
if (rarely()) {
|
||||
sb.append(randomFrom(Arrays.asList("-")));
|
||||
}
|
||||
}
|
||||
}
|
||||
sb.append(".");
|
||||
}
|
||||
sb.deleteCharAt(sb.length() - 1); // remove trailing dot
|
||||
}
|
||||
// optional build part
|
||||
if (randomBoolean()) {
|
||||
sb.append("+").append(randomAlphaOfLengthBetween(1, 15));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* taken from https://regex101.com/r/vkijKf/1/ via https://semver.org/
|
||||
*/
|
||||
public void testSemVerValidation() {
|
||||
String[] validSemverVersions = new String[] {
|
||||
"0.0.4",
|
||||
"1.2.3",
|
||||
"10.20.30",
|
||||
"1.1.2-prerelease+meta",
|
||||
"1.1.2+meta",
|
||||
"1.1.2+meta-valid",
|
||||
"1.0.0-alpha",
|
||||
"1.0.0-beta",
|
||||
"1.0.0-alpha.beta",
|
||||
"1.0.0-alpha.beta.1",
|
||||
"1.0.0-alpha.1",
|
||||
"1.0.0-alpha0.valid",
|
||||
"1.0.0-alpha.0valid",
|
||||
"1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay",
|
||||
"1.0.0-rc.1+build.1",
|
||||
"2.0.0-rc.1+build.123",
|
||||
"1.2.3-beta",
|
||||
"10.2.3-DEV-SNAPSHOT",
|
||||
"1.2.3-SNAPSHOT-123",
|
||||
"1.0.0",
|
||||
"2.0.0",
|
||||
"1.1.7",
|
||||
"2.0.0+build.1848",
|
||||
"2.0.1-alpha.1227",
|
||||
"1.0.0-alpha+beta",
|
||||
"1.2.3----RC-SNAPSHOT.12.9.1--.12+788",
|
||||
"1.2.3----R-S.12.9.1--.12+meta",
|
||||
"1.2.3----RC-SNAPSHOT.12.9.1--.12",
|
||||
"1.0.0+0.build.1-rc.10000aaa-kk-0.1",
|
||||
|
||||
"999999999.999999999.999999999",
|
||||
"1.0.0-0A.is.legal",
|
||||
// the following are not strict semver but we allow them
|
||||
"1.2-SNAPSHOT",
|
||||
"1.2-RC-SNAPSHOT",
|
||||
"1",
|
||||
"1.2.3.4" };
|
||||
for (String version : validSemverVersions) {
|
||||
assertTrue("should be valid: " + version, VersionEncoder.encodeVersion(version).isLegal);
|
||||
// since we're here, also check encoding / decoding rountrip
|
||||
assertEquals(version, decodeVersion(encodeVersion(version)));
|
||||
}
|
||||
|
||||
String[] invalidSemverVersions = new String[] {
|
||||
"",
|
||||
"1.2.3-0123",
|
||||
"1.2.3-0123.0123",
|
||||
"1.1.2+.123",
|
||||
"+invalid",
|
||||
"-invalid",
|
||||
"-invalid+invalid",
|
||||
"-invalid.01",
|
||||
"alpha",
|
||||
"alpha.beta",
|
||||
"alpha.beta.1",
|
||||
"alpha.1",
|
||||
"alpha+beta",
|
||||
"alpha_beta",
|
||||
"alpha.",
|
||||
"alpha..",
|
||||
"beta",
|
||||
"1.0.0-alpha_beta",
|
||||
"-alpha.",
|
||||
"1.0.0-alpha..",
|
||||
"1.0.0-alpha..1",
|
||||
"1.0.0-alpha...1",
|
||||
"1.0.0-alpha....1",
|
||||
"1.0.0-alpha.....1",
|
||||
"1.0.0-alpha......1",
|
||||
"1.0.0-alpha.......1",
|
||||
"01.1.1",
|
||||
"1.01.1",
|
||||
"1.1.01",
|
||||
"1.2.3.DEV",
|
||||
"1.2.31.2.3----RC-SNAPSHOT.12.09.1--..12+788",
|
||||
"-1.0.3-gamma+b7718",
|
||||
"+justmeta",
|
||||
"9.8.7+meta+meta",
|
||||
"9.8.7-whatever+meta+meta",
|
||||
"999999999.999999999.999999999.----RC-SNAPSHOT.12.09.1--------------------------------..12",
|
||||
"12.el2",
|
||||
"12.el2-1.0-rc5",
|
||||
"6.nüll.7" // make sure extended ascii-range (128-255) in invalid versions is decoded correctly
|
||||
};
|
||||
for (String version : invalidSemverVersions) {
|
||||
assertFalse("should be invalid: " + version, VersionEncoder.encodeVersion(version).isLegal);
|
||||
// since we're here, also check encoding / decoding rountrip
|
||||
assertEquals(version, decodeVersion(encodeVersion(version)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,139 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.versionfield;
|
||||
|
||||
import org.apache.lucene.index.DocValuesType;
|
||||
import org.apache.lucene.index.IndexOptions;
|
||||
import org.apache.lucene.index.IndexableField;
|
||||
import org.apache.lucene.index.IndexableFieldType;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.cluster.metadata.IndexMetadata;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.bytes.BytesReference;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.common.xcontent.XContentType;
|
||||
import org.elasticsearch.index.mapper.ContentPath;
|
||||
import org.elasticsearch.index.mapper.DocumentMapper;
|
||||
import org.elasticsearch.index.mapper.Mapper;
|
||||
import org.elasticsearch.index.mapper.MapperParsingException;
|
||||
import org.elasticsearch.index.mapper.MapperTestCase;
|
||||
import org.elasticsearch.index.mapper.ParsedDocument;
|
||||
import org.elasticsearch.index.mapper.SourceToParse;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
|
||||
public class VersionStringFieldMapperTests extends MapperTestCase {
|
||||
|
||||
@Override
|
||||
protected Collection<? extends Plugin> getPlugins() {
|
||||
return Collections.singletonList(new VersionFieldPlugin(getIndexSettings()));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void minimalMapping(XContentBuilder b) throws IOException {
|
||||
b.field("type", "version");
|
||||
}
|
||||
|
||||
public void testDefaults() throws Exception {
|
||||
XContentBuilder mapping = fieldMapping(this::minimalMapping);
|
||||
DocumentMapper mapper = createDocumentMapper(mapping);
|
||||
assertEquals(Strings.toString(mapping), mapper.mappingSource().toString());
|
||||
|
||||
ParsedDocument doc = mapper.parse(
|
||||
new SourceToParse(
|
||||
"test",
|
||||
"_doc",
|
||||
"1",
|
||||
BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "1.2.3").endObject()),
|
||||
XContentType.JSON
|
||||
)
|
||||
);
|
||||
|
||||
IndexableField[] fields = doc.rootDoc().getFields("field");
|
||||
assertEquals(2, fields.length);
|
||||
|
||||
assertEquals("1.2.3", VersionEncoder.decodeVersion(fields[0].binaryValue()));
|
||||
IndexableFieldType fieldType = fields[0].fieldType();
|
||||
assertThat(fieldType.omitNorms(), equalTo(true));
|
||||
assertFalse(fieldType.tokenized());
|
||||
assertFalse(fieldType.stored());
|
||||
assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
|
||||
assertThat(fieldType.storeTermVectors(), equalTo(false));
|
||||
assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
|
||||
assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
|
||||
assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
|
||||
assertEquals(DocValuesType.NONE, fieldType.docValuesType());
|
||||
|
||||
assertEquals("1.2.3", VersionEncoder.decodeVersion(fields[1].binaryValue()));
|
||||
fieldType = fields[1].fieldType();
|
||||
assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
|
||||
assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
|
||||
|
||||
}
|
||||
|
||||
public void testParsesNestedEmptyObjectStrict() throws IOException {
|
||||
DocumentMapper defaultMapper = createDocumentMapper(fieldMapping(this::minimalMapping));
|
||||
|
||||
BytesReference source = BytesReference.bytes(
|
||||
XContentFactory.jsonBuilder().startObject().startObject("field").endObject().endObject()
|
||||
);
|
||||
MapperParsingException ex = expectThrows(
|
||||
MapperParsingException.class,
|
||||
() -> defaultMapper.parse(new SourceToParse("test", "_doc", "1", source, XContentType.JSON))
|
||||
);
|
||||
assertEquals(
|
||||
"failed to parse field [field] of type [version] in document with id '1'. " + "Preview of field's value: '{}'",
|
||||
ex.getMessage()
|
||||
);
|
||||
}
|
||||
|
||||
public void testFailsParsingNestedList() throws IOException {
|
||||
DocumentMapper defaultMapper = createDocumentMapper(fieldMapping(this::minimalMapping));
|
||||
BytesReference source = BytesReference.bytes(
|
||||
XContentFactory.jsonBuilder()
|
||||
.startObject()
|
||||
.startArray("field")
|
||||
.startObject()
|
||||
.startArray("array_name")
|
||||
.value("inner_field_first")
|
||||
.value("inner_field_second")
|
||||
.endArray()
|
||||
.endObject()
|
||||
.endArray()
|
||||
.endObject()
|
||||
);
|
||||
MapperParsingException ex = expectThrows(
|
||||
MapperParsingException.class,
|
||||
() -> defaultMapper.parse(new SourceToParse("test", "_doc", "1", source, XContentType.JSON))
|
||||
);
|
||||
assertEquals(
|
||||
"failed to parse field [field] of type [version] in document with id '1'. "
|
||||
+ "Preview of field's value: '{array_name=[inner_field_first, inner_field_second]}'",
|
||||
ex.getMessage()
|
||||
);
|
||||
}
|
||||
|
||||
public void testFetchSourceValue() throws IOException {
|
||||
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
|
||||
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
|
||||
|
||||
VersionStringFieldMapper mapper = new VersionStringFieldMapper.Builder("field").build(context);
|
||||
assertEquals(org.elasticsearch.common.collect.List.of("value"), fetchSourceValue(mapper, "value"));
|
||||
assertEquals(org.elasticsearch.common.collect.List.of("42"), fetchSourceValue(mapper, 42L));
|
||||
assertEquals(org.elasticsearch.common.collect.List.of("true"), fetchSourceValue(mapper, true));
|
||||
|
||||
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> fetchSourceValue(mapper, "value", "format"));
|
||||
assertEquals("Field [field] of type [version] doesn't support formats.", e.getMessage());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,533 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.versionfield;
|
||||
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.search.SearchHit;
|
||||
import org.elasticsearch.search.aggregations.AggregationBuilders;
|
||||
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
|
||||
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
|
||||
import org.elasticsearch.search.aggregations.metrics.Cardinality;
|
||||
import org.elasticsearch.search.sort.SortOrder;
|
||||
import org.elasticsearch.test.ESSingleNodeTestCase;
|
||||
import org.elasticsearch.xpack.analytics.AnalyticsAggregationBuilders;
|
||||
import org.elasticsearch.xpack.analytics.AnalyticsPlugin;
|
||||
import org.elasticsearch.xpack.analytics.stringstats.InternalStringStats;
|
||||
import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
||||
|
||||
public class VersionStringFieldTests extends ESSingleNodeTestCase {
|
||||
|
||||
@Override
|
||||
protected Collection<Class<? extends Plugin>> getPlugins() {
|
||||
return pluginList(VersionFieldPlugin.class, LocalStateCompositeXPackPlugin.class, AnalyticsPlugin.class);
|
||||
}
|
||||
|
||||
public String setUpIndex(String indexName) throws IOException {
|
||||
createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version");
|
||||
ensureGreen(indexName);
|
||||
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("1")
|
||||
.setSource(jsonBuilder().startObject().field("version", "11.1.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("2")
|
||||
.setSource(jsonBuilder().startObject().field("version", "1.0.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("3")
|
||||
.setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("4")
|
||||
.setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("5")
|
||||
.setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("6")
|
||||
.setSource(jsonBuilder().startObject().field("version", "21.11.0").endObject())
|
||||
.get();
|
||||
client().admin().indices().prepareRefresh(indexName).get();
|
||||
return indexName;
|
||||
}
|
||||
|
||||
public void testExactQueries() throws Exception {
|
||||
String indexName = "test";
|
||||
setUpIndex(indexName);
|
||||
|
||||
// match
|
||||
SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", ("1.0.0"))).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.4.0")).get();
|
||||
assertEquals(0, response.getHits().getTotalHits().value);
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.3.0")).get();
|
||||
assertEquals(0, response.getHits().getTotalHits().value);
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.3.0+build.1234567")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
|
||||
// term
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.0.0")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.4.0")).get();
|
||||
assertEquals(0, response.getHits().getTotalHits().value);
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.3.0")).get();
|
||||
assertEquals(0, response.getHits().getTotalHits().value);
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.3.0+build.1234567")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
|
||||
// terms
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.termsQuery("version", "1.0.0", "1.3.0")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.termsQuery("version", "1.4.0", "1.3.0+build.1234567")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
|
||||
// phrase query (just for keyword compatibility)
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchPhraseQuery("version", "2.1.0-alpha.beta")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
}
|
||||
|
||||
public void testRangeQueries() throws Exception {
|
||||
String indexName = setUpIndex("test");
|
||||
SearchResponse response = client().prepareSearch(indexName)
|
||||
.setQuery(QueryBuilders.rangeQuery("version").from("1.0.0").to("3.0.0"))
|
||||
.get();
|
||||
assertEquals(4, response.getHits().getTotalHits().value);
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.1.0").to("3.0.0")).get();
|
||||
assertEquals(3, response.getHits().getTotalHits().value);
|
||||
response = client().prepareSearch(indexName)
|
||||
.setQuery(QueryBuilders.rangeQuery("version").from("0.1.0").to("2.1.0-alpha.beta"))
|
||||
.get();
|
||||
assertEquals(3, response.getHits().getTotalHits().value);
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("2.1.0").to("3.0.0")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("3.0.0").to("4.0.0")).get();
|
||||
assertEquals(0, response.getHits().getTotalHits().value);
|
||||
response = client().prepareSearch(indexName)
|
||||
.setQuery(QueryBuilders.rangeQuery("version").from("1.3.0+build.1234569").to("3.0.0"))
|
||||
.get();
|
||||
assertEquals(2, response.getHits().getTotalHits().value);
|
||||
|
||||
// ranges excluding edges
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.0.0", false).to("3.0.0")).get();
|
||||
assertEquals(3, response.getHits().getTotalHits().value);
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.0.0").to("2.1.0", false)).get();
|
||||
assertEquals(3, response.getHits().getTotalHits().value);
|
||||
|
||||
// open ranges
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.4.0")).get();
|
||||
assertEquals(4, response.getHits().getTotalHits().value);
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("1.4.0")).get();
|
||||
assertEquals(2, response.getHits().getTotalHits().value);
|
||||
}
|
||||
|
||||
public void testPrefixQuery() throws IOException {
|
||||
String indexName = setUpIndex("test");
|
||||
// prefix
|
||||
SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "1")).get();
|
||||
assertEquals(3, response.getHits().getTotalHits().value);
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2.1")).get();
|
||||
assertEquals(2, response.getHits().getTotalHits().value);
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2.1.0-")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "1.3.0+b")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2")).get();
|
||||
assertEquals(3, response.getHits().getTotalHits().value);
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21.")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21.1")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21.11")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
}
|
||||
|
||||
public void testSort() throws IOException {
|
||||
String indexName = setUpIndex("test");
|
||||
// also adding some invalid versions that should be sorted after legal ones
|
||||
client().prepareIndex(indexName, "_doc").setSource(jsonBuilder().startObject().field("version", "1.2.3alpha").endObject()).get();
|
||||
client().prepareIndex(indexName, "_doc").setSource(jsonBuilder().startObject().field("version", "1.3.567#12").endObject()).get();
|
||||
client().admin().indices().prepareRefresh(indexName).get();
|
||||
|
||||
// sort based on version field
|
||||
SearchResponse response = client().prepareSearch(indexName)
|
||||
.setQuery(QueryBuilders.matchAllQuery())
|
||||
.addSort("version", SortOrder.DESC)
|
||||
.get();
|
||||
assertEquals(8, response.getHits().getTotalHits().value);
|
||||
SearchHit[] hits = response.getHits().getHits();
|
||||
assertEquals("1.3.567#12", hits[0].getSortValues()[0]);
|
||||
assertEquals("1.2.3alpha", hits[1].getSortValues()[0]);
|
||||
assertEquals("21.11.0", hits[2].getSortValues()[0]);
|
||||
assertEquals("11.1.0", hits[3].getSortValues()[0]);
|
||||
assertEquals("2.1.0", hits[4].getSortValues()[0]);
|
||||
assertEquals("2.1.0-alpha.beta", hits[5].getSortValues()[0]);
|
||||
assertEquals("1.3.0+build.1234567", hits[6].getSortValues()[0]);
|
||||
assertEquals("1.0.0", hits[7].getSortValues()[0]);
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).addSort("version", SortOrder.ASC).get();
|
||||
assertEquals(8, response.getHits().getTotalHits().value);
|
||||
hits = response.getHits().getHits();
|
||||
assertEquals("1.0.0", hits[0].getSortValues()[0]);
|
||||
assertEquals("1.3.0+build.1234567", hits[1].getSortValues()[0]);
|
||||
assertEquals("2.1.0-alpha.beta", hits[2].getSortValues()[0]);
|
||||
assertEquals("2.1.0", hits[3].getSortValues()[0]);
|
||||
assertEquals("11.1.0", hits[4].getSortValues()[0]);
|
||||
assertEquals("21.11.0", hits[5].getSortValues()[0]);
|
||||
assertEquals("1.2.3alpha", hits[6].getSortValues()[0]);
|
||||
assertEquals("1.3.567#12", hits[7].getSortValues()[0]);
|
||||
}
|
||||
|
||||
public void testRegexQuery() throws Exception {
|
||||
String indexName = "test_regex";
|
||||
createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version");
|
||||
ensureGreen(indexName);
|
||||
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("1")
|
||||
.setSource(jsonBuilder().startObject().field("version", "1.0.0alpha2.1.0-rc.1").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("2")
|
||||
.setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("3")
|
||||
.setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("4")
|
||||
.setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("5")
|
||||
.setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject())
|
||||
.get();
|
||||
client().admin().indices().prepareRefresh(indexName).get();
|
||||
|
||||
SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "2.*0")).get();
|
||||
assertEquals(2, response.getHits().getTotalHits().value);
|
||||
assertEquals("2.1.0", response.getHits().getHits()[0].getSourceAsMap().get("version"));
|
||||
assertEquals("2.33.0", response.getHits().getHits()[1].getSourceAsMap().get("version"));
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "<0-10>.<0-10>.*al.*")).get();
|
||||
assertEquals(2, response.getHits().getTotalHits().value);
|
||||
assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version"));
|
||||
assertEquals("2.1.0-alpha.beta", response.getHits().getHits()[1].getSourceAsMap().get("version"));
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "1.[0-9].[0-9].*")).get();
|
||||
assertEquals(2, response.getHits().getTotalHits().value);
|
||||
assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version"));
|
||||
assertEquals("1.3.0+build.1234567", response.getHits().getHits()[1].getSourceAsMap().get("version"));
|
||||
|
||||
// test case sensitivity / insensitivity
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", ".*alpha.*")).get();
|
||||
assertEquals(2, response.getHits().getTotalHits().value);
|
||||
assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version"));
|
||||
assertEquals("2.1.0-alpha.beta", response.getHits().getHits()[1].getSourceAsMap().get("version"));
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", ".*Alpha.*")).get();
|
||||
assertEquals(0, response.getHits().getTotalHits().value);
|
||||
|
||||
response = client().prepareSearch(indexName)
|
||||
.setQuery(QueryBuilders.regexpQuery("version", ".*Alpha.*").caseInsensitive(true))
|
||||
.get();
|
||||
assertEquals(2, response.getHits().getTotalHits().value);
|
||||
assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version"));
|
||||
assertEquals("2.1.0-alpha.beta", response.getHits().getHits()[1].getSourceAsMap().get("version"));
|
||||
|
||||
}
|
||||
|
||||
public void testFuzzyQuery() throws Exception {
|
||||
String indexName = "test_fuzzy";
|
||||
createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version");
|
||||
ensureGreen(indexName);
|
||||
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("1")
|
||||
.setSource(jsonBuilder().startObject().field("version", "1.0.0-alpha.2.1.0-rc.1").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("2")
|
||||
.setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("3")
|
||||
.setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("4")
|
||||
.setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("5")
|
||||
.setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("6")
|
||||
.setSource(jsonBuilder().startObject().field("version", "2.a3.0").endObject())
|
||||
.get();
|
||||
client().admin().indices().prepareRefresh(indexName).get();
|
||||
|
||||
SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.fuzzyQuery("version", "2.3.0")).get();
|
||||
assertEquals(3, response.getHits().getTotalHits().value);
|
||||
assertEquals("2.1.0", response.getHits().getHits()[0].getSourceAsMap().get("version"));
|
||||
assertEquals("2.33.0", response.getHits().getHits()[1].getSourceAsMap().get("version"));
|
||||
assertEquals("2.a3.0", response.getHits().getHits()[2].getSourceAsMap().get("version"));
|
||||
}
|
||||
|
||||
public void testWildcardQuery() throws Exception {
|
||||
String indexName = "test_wildcard";
|
||||
createIndex(
|
||||
indexName,
|
||||
Settings.builder().put("index.number_of_shards", 1).build(),
|
||||
"_doc",
|
||||
"version",
|
||||
"type=version",
|
||||
"foo",
|
||||
"type=keyword"
|
||||
);
|
||||
ensureGreen(indexName);
|
||||
|
||||
for (String version : org.elasticsearch.common.collect.List.of(
|
||||
"1.0.0-alpha.2.1.0-rc.1",
|
||||
"1.3.0+build.1234567",
|
||||
"2.1.0-alpha.beta",
|
||||
"2.1.0",
|
||||
"2.33.0",
|
||||
"3.1.1-a",
|
||||
"3.1.1+b",
|
||||
"3.1.123"
|
||||
)) {
|
||||
client().prepareIndex(indexName, "_doc").setSource(jsonBuilder().startObject().field("version", version).endObject()).get();
|
||||
}
|
||||
client().admin().indices().prepareRefresh(indexName).get();
|
||||
|
||||
checkWildcardQuery(indexName, "*alpha*", new String[] { "1.0.0-alpha.2.1.0-rc.1", "2.1.0-alpha.beta" });
|
||||
checkWildcardQuery(indexName, "*b*", new String[] { "1.3.0+build.1234567", "2.1.0-alpha.beta", "3.1.1+b" });
|
||||
checkWildcardQuery(indexName, "*bet*", new String[] { "2.1.0-alpha.beta" });
|
||||
checkWildcardQuery(indexName, "2.1*", new String[] { "2.1.0-alpha.beta", "2.1.0" });
|
||||
checkWildcardQuery(indexName, "2.1.0-*", new String[] { "2.1.0-alpha.beta" });
|
||||
checkWildcardQuery(indexName, "*2.1.0-*", new String[] { "1.0.0-alpha.2.1.0-rc.1", "2.1.0-alpha.beta" });
|
||||
checkWildcardQuery(indexName, "*2.1.0*", new String[] { "1.0.0-alpha.2.1.0-rc.1", "2.1.0-alpha.beta", "2.1.0" });
|
||||
checkWildcardQuery(indexName, "*2.?.0*", new String[] { "1.0.0-alpha.2.1.0-rc.1", "2.1.0-alpha.beta", "2.1.0" });
|
||||
checkWildcardQuery(indexName, "*2.??.0*", new String[] { "2.33.0" });
|
||||
checkWildcardQuery(indexName, "?.1.0", new String[] { "2.1.0" });
|
||||
checkWildcardQuery(indexName, "*-*", new String[] { "1.0.0-alpha.2.1.0-rc.1", "2.1.0-alpha.beta", "3.1.1-a" });
|
||||
checkWildcardQuery(indexName, "1.3.0+b*", new String[] { "1.3.0+build.1234567" });
|
||||
checkWildcardQuery(indexName, "3.1.1??", new String[] { "3.1.1-a", "3.1.1+b", "3.1.123" });
|
||||
}
|
||||
|
||||
private void checkWildcardQuery(String indexName, String query, String... expectedResults) {
|
||||
SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", query)).get();
|
||||
assertEquals(expectedResults.length, response.getHits().getTotalHits().value);
|
||||
for (int i = 0; i < expectedResults.length; i++) {
|
||||
String expected = expectedResults[i];
|
||||
Object actual = response.getHits().getHits()[i].getSourceAsMap().get("version");
|
||||
assertEquals("expected " + expected + " in position " + i + " but found " + actual, expected, actual);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* test that versions that are invalid under semver are still indexed and retrieveable, though they sort differently
|
||||
*/
|
||||
public void testStoreMalformed() throws Exception {
|
||||
String indexName = "test_malformed";
|
||||
createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version");
|
||||
ensureGreen(indexName);
|
||||
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("1")
|
||||
.setSource(jsonBuilder().startObject().field("version", "1.invalid.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("2")
|
||||
.setSource(jsonBuilder().startObject().field("version", "2.2.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("3")
|
||||
.setSource(jsonBuilder().startObject().field("version", "2.2.0-badchar!").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc").setId("4").setSource(jsonBuilder().startObject().field("version", "").endObject()).get();
|
||||
client().admin().indices().prepareRefresh(indexName).get();
|
||||
|
||||
SearchResponse response = client().prepareSearch(indexName).addDocValueField("version").get();
|
||||
assertEquals(4, response.getHits().getTotalHits().value);
|
||||
assertEquals("1", response.getHits().getAt(0).getId());
|
||||
assertEquals("1.invalid.0", response.getHits().getAt(0).field("version").getValue());
|
||||
|
||||
assertEquals("2", response.getHits().getAt(1).getId());
|
||||
assertEquals("2.2.0", response.getHits().getAt(1).field("version").getValue());
|
||||
|
||||
assertEquals("3", response.getHits().getAt(2).getId());
|
||||
assertEquals("2.2.0-badchar!", response.getHits().getAt(2).field("version").getValue());
|
||||
|
||||
assertEquals("4", response.getHits().getAt(3).getId());
|
||||
assertEquals("", response.getHits().getAt(3).field("version").getValue());
|
||||
|
||||
// exact match for malformed term
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.invalid.0")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "2.2.0-badchar!")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
|
||||
// also should appear in terms aggs
|
||||
response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version")).get();
|
||||
Terms terms = response.getAggregations().get("myterms");
|
||||
List<? extends Bucket> buckets = terms.getBuckets();
|
||||
|
||||
assertEquals(4, buckets.size());
|
||||
assertEquals("2.2.0", buckets.get(0).getKey());
|
||||
assertEquals("", buckets.get(1).getKey());
|
||||
assertEquals("1.invalid.0", buckets.get(2).getKey());
|
||||
assertEquals("2.2.0-badchar!", buckets.get(3).getKey());
|
||||
|
||||
// invalid values should sort after all valid ones
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).addSort("version", SortOrder.ASC).get();
|
||||
assertEquals(4, response.getHits().getTotalHits().value);
|
||||
SearchHit[] hits = response.getHits().getHits();
|
||||
assertEquals("2.2.0", hits[0].getSortValues()[0]);
|
||||
assertEquals("", hits[1].getSortValues()[0]);
|
||||
assertEquals("1.invalid.0", hits[2].getSortValues()[0]);
|
||||
assertEquals("2.2.0-badchar!", hits[3].getSortValues()[0]);
|
||||
|
||||
// ranges can include them, but they are sorted last
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("3.0.0")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("3.0.0")).get();
|
||||
assertEquals(3, response.getHits().getTotalHits().value);
|
||||
|
||||
// using the empty string as lower bound should return all "invalid" versions
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("")).get();
|
||||
assertEquals(3, response.getHits().getTotalHits().value);
|
||||
}
|
||||
|
||||
public void testAggs() throws Exception {
|
||||
String indexName = "test_aggs";
|
||||
createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version");
|
||||
ensureGreen(indexName);
|
||||
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("1")
|
||||
.setSource(jsonBuilder().startObject().field("version", "1.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("2")
|
||||
.setSource(jsonBuilder().startObject().field("version", "1.3.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("3")
|
||||
.setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("4")
|
||||
.setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("5")
|
||||
.setSource(jsonBuilder().startObject().field("version", "3.11.5").endObject())
|
||||
.get();
|
||||
client().admin().indices().prepareRefresh(indexName).get();
|
||||
|
||||
// terms aggs
|
||||
SearchResponse response = client().prepareSearch(indexName)
|
||||
.addAggregation(AggregationBuilders.terms("myterms").field("version"))
|
||||
.get();
|
||||
Terms terms = response.getAggregations().get("myterms");
|
||||
List<? extends Bucket> buckets = terms.getBuckets();
|
||||
|
||||
assertEquals(5, buckets.size());
|
||||
assertEquals("1.0", buckets.get(0).getKey());
|
||||
assertEquals("1.3.0", buckets.get(1).getKey());
|
||||
assertEquals("2.1.0-alpha", buckets.get(2).getKey());
|
||||
assertEquals("2.1.0", buckets.get(3).getKey());
|
||||
assertEquals("3.11.5", buckets.get(4).getKey());
|
||||
|
||||
// cardinality
|
||||
response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.cardinality("myterms").field("version")).get();
|
||||
Cardinality card = response.getAggregations().get("myterms");
|
||||
assertEquals(5, card.getValue());
|
||||
|
||||
// string stats
|
||||
response = client().prepareSearch(indexName)
|
||||
.addAggregation(AnalyticsAggregationBuilders.stringStats("stats").field("version"))
|
||||
.get();
|
||||
InternalStringStats stats = response.getAggregations().get("stats");
|
||||
assertEquals(3, stats.getMinLength());
|
||||
assertEquals(11, stats.getMaxLength());
|
||||
}
|
||||
|
||||
public void testMultiValues() throws Exception {
|
||||
String indexName = "test_multi";
|
||||
createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version");
|
||||
ensureGreen(indexName);
|
||||
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("1")
|
||||
.setSource(jsonBuilder().startObject().array("version", "1.0.0", "3.0.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("2")
|
||||
.setSource(jsonBuilder().startObject().array("version", "2.0.0", "4.alpha.0").endObject())
|
||||
.get();
|
||||
client().prepareIndex(indexName, "_doc")
|
||||
.setId("3")
|
||||
.setSource(jsonBuilder().startObject().array("version", "2.1.0", "2.2.0", "5.99.0").endObject())
|
||||
.get();
|
||||
client().admin().indices().prepareRefresh(indexName).get();
|
||||
|
||||
SearchResponse response = client().prepareSearch(indexName).addSort("version", SortOrder.ASC).get();
|
||||
assertEquals(3, response.getHits().getTotalHits().value);
|
||||
assertEquals("1", response.getHits().getAt(0).getId());
|
||||
assertEquals("2", response.getHits().getAt(1).getId());
|
||||
assertEquals("3", response.getHits().getAt(2).getId());
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "3.0.0")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
assertEquals("1", response.getHits().getAt(0).getId());
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "4.alpha.0")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
assertEquals("2", response.getHits().getAt(0).getId());
|
||||
|
||||
// range
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("1.5.0")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.5.0")).get();
|
||||
assertEquals(3, response.getHits().getTotalHits().value);
|
||||
|
||||
response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("5.0.0").to("6.0.0")).get();
|
||||
assertEquals(1, response.getHits().getTotalHits().value);
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user