honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SQL: Support pattern against compatible indices (#34718) 

Extend querying support on multiple indices from being strictly
identical to being just compatible.
Use FieldCapabilities API (extended through #33803) for mapping merging.

Close #31837 #31611
This commit is contained in:
Costin Leau 2018-10-23 17:07:51 +03:00 committed by GitHub
parent 36baf3823d
commit ca6808e55d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 733 additions and 354 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/reference/sql/security.asciidoc
View File

@ -34,6 +34,6 @@ indices:
["source","yaml",subs="attributes,callouts,macros"] ["source","yaml",subs="attributes,callouts,macros"]
-------------------------------------------------- --------------------------------------------------
include-tagged::{sql-tests}/security/roles.yml[cli_jdbc] include-tagged::{sql-tests}/security/roles.yml[cli_drivers]
-------------------------------------------------- --------------------------------------------------

8
x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java
View File

@ -36,8 +36,8 @@ public enum DataType {
SCALED_FLOAT(JDBCType.FLOAT, Double.class, Double.BYTES, 19, 25, false, true, true), SCALED_FLOAT(JDBCType.FLOAT, Double.class, Double.BYTES, 19, 25, false, true, true),
KEYWORD( JDBCType.VARCHAR, String.class, Integer.MAX_VALUE, 256, 0), KEYWORD( JDBCType.VARCHAR, String.class, Integer.MAX_VALUE, 256, 0),
TEXT( JDBCType.VARCHAR, String.class, Integer.MAX_VALUE, Integer.MAX_VALUE, 0, false, false, false), TEXT( JDBCType.VARCHAR, String.class, Integer.MAX_VALUE, Integer.MAX_VALUE, 0, false, false, false),
OBJECT( JDBCType.STRUCT, null, -1, 0, 0), OBJECT( JDBCType.STRUCT, null, -1, 0, 0, false, false, false),
NESTED( JDBCType.STRUCT, null, -1, 0, 0), NESTED( JDBCType.STRUCT, null, -1, 0, 0, false, false, false),
BINARY( JDBCType.VARBINARY, byte[].class, -1, Integer.MAX_VALUE, 0), BINARY( JDBCType.VARBINARY, byte[].class, -1, Integer.MAX_VALUE, 0),
// since ODBC and JDBC interpret precision for Date as display size, // since ODBC and JDBC interpret precision for Date as display size,
// the precision is 23 (number of chars in ISO8601 with millis) + Z (the UTC timezone) // the precision is 23 (number of chars in ISO8601 with millis) + Z (the UTC timezone)
@ -223,7 +223,11 @@ public enum DataType {
* For any dataType DataType.fromEsType(dataType.esType) == dataType * For any dataType DataType.fromEsType(dataType.esType) == dataType
*/ */
public static DataType fromEsType(String esType) { public static DataType fromEsType(String esType) {
try {
return DataType.valueOf(esType.toUpperCase(Locale.ROOT)); return DataType.valueOf(esType.toUpperCase(Locale.ROOT));
} catch (IllegalArgumentException ex) {
return DataType.UNSUPPORTED;
}
} }
public boolean isCompatibleWith(DataType other) { public boolean isCompatibleWith(DataType other) {

194
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java
View File

@ -15,6 +15,8 @@ import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest.Feature; import org.elasticsearch.action.admin.indices.get.GetIndexRequest.Feature;
import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
import org.elasticsearch.action.fieldcaps.FieldCapabilities;
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.IndicesOptions.Option; import org.elasticsearch.action.support.IndicesOptions.Option;
import org.elasticsearch.action.support.IndicesOptions.WildcardStates; import org.elasticsearch.action.support.IndicesOptions.WildcardStates;
@ -24,23 +26,34 @@ import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
import org.elasticsearch.xpack.sql.type.DataType;
import org.elasticsearch.xpack.sql.type.DateEsField;
import org.elasticsearch.xpack.sql.type.EsField; import org.elasticsearch.xpack.sql.type.EsField;
import org.elasticsearch.xpack.sql.type.KeywordEsField;
import org.elasticsearch.xpack.sql.type.TextEsField;
import org.elasticsearch.xpack.sql.type.Types; import org.elasticsearch.xpack.sql.type.Types;
import org.elasticsearch.xpack.sql.type.UnsupportedEsField;
import org.elasticsearch.xpack.sql.util.CollectionUtils; import org.elasticsearch.xpack.sql.util.CollectionUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry;
import java.util.NavigableSet;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet; import java.util.TreeSet;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap;
public class IndexResolver { public class IndexResolver {
@ -222,64 +235,157 @@ public class IndexResolver {
listener.onResponse(result); listener.onResponse(result);
} }
/** /**
* Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping. * Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping.
*/ */
public void resolveWithSameMapping(String indexWildcard, String javaRegex, ActionListener<IndexResolution> listener) { public void resolveAsMergedMapping(String indexWildcard, String javaRegex, ActionListener<IndexResolution> listener) {
GetIndexRequest getIndexRequest = createGetIndexRequest(indexWildcard); FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard);
client.admin().indices().getIndex(getIndexRequest, ActionListener.wrap(response -> { client.fieldCaps(fieldRequest,
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = response.getMappings(); ActionListener.wrap(response -> listener.onResponse(mergedMapping(indexWildcard, response.get())), listener::onFailure));
List<IndexResolution> resolutions;
if (mappings.size() > 0) {
resolutions = new ArrayList<>(mappings.size());
Pattern pattern = javaRegex != null ? Pattern.compile(javaRegex) : null;
for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> indexMappings : mappings) {
String concreteIndex = indexMappings.key;
if (pattern == null || pattern.matcher(concreteIndex).matches()) {
resolutions.add(buildGetIndexResult(concreteIndex, concreteIndex, indexMappings.value));
}
}
} else {
resolutions = emptyList();
} }
listener.onResponse(merge(resolutions, indexWildcard)); static IndexResolution mergedMapping(String indexPattern, Map<String, Map<String, FieldCapabilities>> fieldCaps) {
}, listener::onFailure)); if (fieldCaps == null || fieldCaps.isEmpty()) {
return IndexResolution.notFound(indexPattern);
} }
static IndexResolution merge(List<IndexResolution> resolutions, String indexWildcard) { StringBuilder errorMessage = new StringBuilder();
IndexResolution merged = null;
for (IndexResolution resolution : resolutions) { NavigableSet<Entry<String, Map<String, FieldCapabilities>>> sortedFields = new TreeSet<>(
// everything that follows gets compared // for some reason .reversed doesn't work (prolly due to inference)
if (!resolution.isValid()) { Collections.reverseOrder(Comparator.comparing(Entry::getKey)));
return resolution; sortedFields.addAll(fieldCaps.entrySet());
Map<String, EsField> hierarchicalMapping = new TreeMap<>();
Map<String, EsField> flattedMapping = new LinkedHashMap<>();
// sort keys descending in order to easily detect multi-fields (a.b.c multi-field of a.b)
// without sorting, they can still be detected however without the emptyMap optimization
// (fields without multi-fields have no children)
for (Entry<String, Map<String, FieldCapabilities>> entry : sortedFields) {
String name = entry.getKey();
// skip internal fields
if (!name.startsWith("_")) {
Map<String, FieldCapabilities> types = entry.getValue();
// field is mapped differently across indices
if (types.size() > 1) {
// build error message
for (Entry<String, FieldCapabilities> type : types.entrySet()) {
if (errorMessage.length() > 0) {
errorMessage.append(", ");
} }
// initialize resolution on first run errorMessage.append("[");
if (merged == null) { errorMessage.append(type.getKey());
merged = resolution; errorMessage.append("] in ");
errorMessage.append(Arrays.toString(type.getValue().indices()));
} }
// need the same mapping across all resolutions
if (!merged.get().mapping().equals(resolution.get().mapping())) { errorMessage.insert(0,
return IndexResolution.invalid( "[" + indexPattern + "] points to indices with incompatible mappings; " +
"[" + indexWildcard + "] points to indices [" + merged.get().name() + "] " "field [" + name + "] is mapped in [" + types.size() + "] different ways: ");
+ "and [" + resolution.get().name() + "] which have different mappings. " }
+ "When using multiple indices, the mappings must be identical."); if (errorMessage.length() > 0) {
return IndexResolution.invalid(errorMessage.toString());
}
FieldCapabilities fieldCap = types.values().iterator().next();
// validate search/agg-able
if (fieldCap.isAggregatable() && fieldCap.nonAggregatableIndices() != null) {
errorMessage.append("[" + indexPattern + "] points to indices with incompatible mappings: ");
errorMessage.append("field [" + name + "] is aggregateable except in ");
errorMessage.append(Arrays.toString(fieldCap.nonAggregatableIndices()));
}
if (fieldCap.isSearchable() && fieldCap.nonSearchableIndices() != null) {
if (errorMessage.length() > 0) {
errorMessage.append(",");
}
errorMessage.append("[" + indexPattern + "] points to indices with incompatible mappings: ");
errorMessage.append("field [" + name + "] is searchable except in ");
errorMessage.append(Arrays.toString(fieldCap.nonSearchableIndices()));
}
if (errorMessage.length() > 0) {
return IndexResolution.invalid(errorMessage.toString());
}
// validation passes - create the field
// and name wasn't added before
if (!flattedMapping.containsKey(name)) {
createField(name, fieldCap, fieldCaps, hierarchicalMapping, flattedMapping, false);
} }
} }
if (merged != null) {
// at this point, we are sure there's the same mapping across all (if that's the case) indices
// to keep things simple, use the given pattern as index name
merged = IndexResolution.valid(new EsIndex(indexWildcard, merged.get().mapping()));
} else {
merged = IndexResolution.notFound(indexWildcard);
} }
return merged;
return IndexResolution.valid(new EsIndex(indexPattern, hierarchicalMapping));
} }
private static EsField createField(String fieldName, FieldCapabilities caps, Map<String, Map<String, FieldCapabilities>> globalCaps,
Map<String, EsField> hierarchicalMapping, Map<String, EsField> flattedMapping, boolean hasChildren) {
Map<String, EsField> parentProps = hierarchicalMapping;
int dot = fieldName.lastIndexOf('.');
String fullFieldName = fieldName;
if (dot >= 0) {
String parentName = fieldName.substring(0, dot);
fieldName = fieldName.substring(dot + 1);
EsField parent = flattedMapping.get(parentName);
if (parent == null) {
Map<String, FieldCapabilities> map = globalCaps.get(parentName);
if (map == null) {
throw new SqlIllegalArgumentException("Cannot find field {}; this is likely a bug", parentName);
}
FieldCapabilities parentCap = map.values().iterator().next();
parent = createField(parentName, parentCap, globalCaps, hierarchicalMapping, flattedMapping, true);
}
parentProps = parent.getProperties();
}
EsField field = null;
Map<String, EsField> props = hasChildren ? new TreeMap<>() : emptyMap();
DataType esType = DataType.fromEsType(caps.getType());
switch (esType) {
case TEXT:
field = new TextEsField(fieldName, props, false);
break;
case KEYWORD:
int length = DataType.KEYWORD.defaultPrecision;
// TODO: to check whether isSearchable/isAggregateable takes into account the presence of the normalizer
boolean normalized = false;
field = new KeywordEsField(fieldName, props, caps.isAggregatable(), length, normalized);
break;
case DATE:
field = new DateEsField(fieldName, props, caps.isAggregatable());
break;
case UNSUPPORTED:
field = new UnsupportedEsField(fieldName, caps.getType());
break;
default:
field = new EsField(fieldName, esType, props, caps.isAggregatable());
}
parentProps.put(fieldName, field);
flattedMapping.put(fullFieldName, field);
return field;
}
private static FieldCapabilitiesRequest createFieldCapsRequest(String index) {
return new FieldCapabilitiesRequest()
.indices(Strings.commaDelimitedListToStringArray(index))
.fields("*")
//lenient because we throw our own errors looking at the response e.g. if something was not resolved
//also because this way security doesn't throw authorization exceptions but rather honors ignore_unavailable
.indicesOptions(IndicesOptions.lenientExpandOpen());
}
// TODO: Concrete indices still uses get mapping
// waiting on https://github.com/elastic/elasticsearch/pull/34071
//
/** /**
* Resolves a pattern to multiple, separate indices. * Resolves a pattern to multiple, separate indices. Doesn't perform validation.
*/ */
public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, ActionListener<List<EsIndex>> listener) { public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, ActionListener<List<EsIndex>> listener) {
GetIndexRequest getIndexRequest = createGetIndexRequest(indexWildcard); GetIndexRequest getIndexRequest = createGetIndexRequest(indexWildcard);

11
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java
View File

@ -54,13 +54,15 @@ public class ShowColumns extends Command {
@Override @Override
public List<Attribute> output() { public List<Attribute> output() {
return asList(new FieldAttribute(location(), "column", new KeywordEsField("column")), return asList(new FieldAttribute(location(), "column", new KeywordEsField("column")),
new FieldAttribute(location(), "type", new KeywordEsField("type"))); } new FieldAttribute(location(), "type", new KeywordEsField("type")),
new FieldAttribute(location(), "mapping", new KeywordEsField("mapping")));
}
@Override @Override
public void execute(SqlSession session, ActionListener<SchemaRowSet> listener) { public void execute(SqlSession session, ActionListener<SchemaRowSet> listener) {
String idx = index != null ? index : (pattern != null ? pattern.asIndexNameWildcard() : "*"); String idx = index != null ? index : (pattern != null ? pattern.asIndexNameWildcard() : "*");
String regex = pattern != null ? pattern.asJavaRegex() : null; String regex = pattern != null ? pattern.asJavaRegex() : null;
session.indexResolver().resolveWithSameMapping(idx, regex, ActionListener.wrap( session.indexResolver().resolveAsMergedMapping(idx, regex, ActionListener.wrap(
indexResult -> { indexResult -> {
List<List<?>> rows = emptyList(); List<List<?>> rows = emptyList();
if (indexResult.isValid()) { if (indexResult.isValid()) {
@ -69,8 +71,7 @@ public class ShowColumns extends Command {
} }
listener.onResponse(Rows.of(output(), rows)); listener.onResponse(Rows.of(output(), rows));
}, },
listener::onFailure listener::onFailure));
));
} }
private void fillInRows(Map<String, EsField> mapping, String prefix, List<List<?>> rows) { private void fillInRows(Map<String, EsField> mapping, String prefix, List<List<?>> rows) {
@ -79,7 +80,7 @@ public class ShowColumns extends Command {
DataType dt = field.getDataType(); DataType dt = field.getDataType();
String name = e.getKey(); String name = e.getKey();
if (dt != null) { if (dt != null) {
rows.add(asList(prefix != null ? prefix + "." + name : name, dt.sqlName())); rows.add(asList(prefix != null ? prefix + "." + name : name, dt.sqlName(), dt.name()));
if (field.getProperties().isEmpty() == false) { if (field.getProperties().isEmpty() == false) {
String newPrefix = prefix != null ? prefix + "." + name : name; String newPrefix = prefix != null ? prefix + "." + name : name;
fillInRows(field.getProperties(), newPrefix, rows); fillInRows(field.getProperties(), newPrefix, rows);

6
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java
View File

@ -172,7 +172,7 @@ public class QueryContainer {
// reference methods // reference methods
// //
private FieldExtraction topHitFieldRef(FieldAttribute fieldAttr) { private FieldExtraction topHitFieldRef(FieldAttribute fieldAttr) {
return new SearchHitFieldRef(aliasName(fieldAttr), fieldAttr.field().getDataType(), fieldAttr.field().hasDocValues()); return new SearchHitFieldRef(aliasName(fieldAttr), fieldAttr.field().getDataType(), fieldAttr.field().isAggregatable());
} }
private Tuple<QueryContainer, FieldExtraction> nestedHitFieldRef(FieldAttribute attr) { private Tuple<QueryContainer, FieldExtraction> nestedHitFieldRef(FieldAttribute attr) {
@ -181,10 +181,10 @@ public class QueryContainer {
String name = aliasName(attr); String name = aliasName(attr);
Query q = rewriteToContainNestedField(query, attr.location(), Query q = rewriteToContainNestedField(query, attr.location(),
attr.nestedParent().name(), name, attr.field().hasDocValues()); attr.nestedParent().name(), name, attr.field().isAggregatable());
SearchHitFieldRef nestedFieldRef = new SearchHitFieldRef(name, attr.field().getDataType(), SearchHitFieldRef nestedFieldRef = new SearchHitFieldRef(name, attr.field().getDataType(),
attr.field().hasDocValues(), attr.parent().name()); attr.field().isAggregatable(), attr.parent().name());
nestedRefs.add(nestedFieldRef); nestedRefs.add(nestedFieldRef);
return new Tuple<>(new QueryContainer(q, aggs, columns, aliases, pseudoFunctions, scalarFunctions, sort, limit), nestedFieldRef); return new Tuple<>(new QueryContainer(q, aggs, columns, aliases, pseudoFunctions, scalarFunctions, sort, limit), nestedFieldRef);

2
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java
View File

@ -127,7 +127,7 @@ public class SqlSession {
listener.onFailure(new MappingException("Cannot inspect indices in cluster/catalog [{}]", cluster)); listener.onFailure(new MappingException("Cannot inspect indices in cluster/catalog [{}]", cluster));
} }
indexResolver.resolveWithSameMapping(table.index(), null, indexResolver.resolveAsMergedMapping(table.index(), null,
wrap(indexResult -> listener.onResponse(action.apply(indexResult)), listener::onFailure)); wrap(indexResult -> listener.onResponse(action.apply(indexResult)), listener::onFailure));
} else { } else {
try { try {

34
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/EsField.java
View File

@ -15,14 +15,14 @@ import java.util.Objects;
*/ */
public class EsField { public class EsField {
private final DataType esDataType; private final DataType esDataType;
private final boolean hasDocValues; private final boolean aggregatable;
private final Map<String, EsField> properties; private final Map<String, EsField> properties;
private final String name; private final String name;
public EsField(String name, DataType esDataType, Map<String, EsField> properties, boolean hasDocValues) { public EsField(String name, DataType esDataType, Map<String, EsField> properties, boolean aggregatable) {
this.name = name; this.name = name;
this.esDataType = esDataType; this.esDataType = esDataType;
this.hasDocValues = hasDocValues; this.aggregatable = aggregatable;
this.properties = properties; this.properties = properties;
} }
@ -41,10 +41,10 @@ public class EsField {
} }
/** /**
* The field supports doc values * This field can be aggregated
*/ */
public boolean hasDocValues() { public boolean isAggregatable() {
return hasDocValues; return aggregatable;
} }
/** /**
@ -85,19 +85,27 @@ public class EsField {
return true; return true;
} }
@Override
public String toString() {
return name + "@" + esDataType.name() + "=" + properties;
}
@Override @Override
public boolean equals(Object o) { public boolean equals(Object o) {
if (this == o) return true; if (this == o) {
if (o == null || getClass() != o.getClass()) return false; return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
EsField field = (EsField) o; EsField field = (EsField) o;
return hasDocValues == field.hasDocValues && return aggregatable == field.aggregatable && esDataType == field.esDataType
esDataType == field.esDataType && && Objects.equals(name, field.name)
Objects.equals(properties, field.properties) && && Objects.equals(properties, field.properties);
Objects.equals(name, field.name);
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(esDataType, hasDocValues, properties, name); return Objects.hash(esDataType, aggregatable, properties, name);
} }
} }

193
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java
View File

@ -5,12 +5,18 @@
*/ */
package org.elasticsearch.xpack.sql.analysis.index; package org.elasticsearch.xpack.sql.analysis.index;
import org.elasticsearch.action.fieldcaps.FieldCapabilities;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.sql.type.DataType;
import org.elasticsearch.xpack.sql.type.EsField; import org.elasticsearch.xpack.sql.type.EsField;
import org.elasticsearch.xpack.sql.type.TypesTests; import org.elasticsearch.xpack.sql.type.TypesTests;
import java.util.Arrays; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry;
public class IndexResolverTests extends ESTestCase { public class IndexResolverTests extends ESTestCase {
@ -21,40 +27,175 @@ public class IndexResolverTests extends ESTestCase {
assertEquals(oneMapping, sameMapping); assertEquals(oneMapping, sameMapping);
String wildcard = "*"; String wildcard = "*";
IndexResolution resolution = IndexResolver.merge(
Arrays.asList(IndexResolution.valid(new EsIndex("a", oneMapping)), IndexResolution.valid(new EsIndex("b", sameMapping))), IndexResolution resolution = IndexResolver.mergedMapping(wildcard, fromMappings(
wildcard); new EsIndex("a", oneMapping),
new EsIndex("b", sameMapping)));
assertTrue(resolution.isValid()); assertTrue(resolution.isValid());
assertEqualsMaps(oneMapping, resolution.get().mapping());
EsIndex esIndex = resolution.get();
assertEquals(wildcard, esIndex.name());
assertEquals(sameMapping, esIndex.mapping());
} }
public void testMergeDifferentMapping() throws Exception { public void testMergeCompatibleMapping() throws Exception {
Map<String, EsField> oneMapping = TypesTests.loadMapping("mapping-basic.json", true); Map<String, EsField> basicMapping = TypesTests.loadMapping("mapping-basic.json", true);
Map<String, EsField> sameMapping = TypesTests.loadMapping("mapping-basic.json", true); Map<String, EsField> numericMapping = TypesTests.loadMapping("mapping-numeric.json", true);
Map<String, EsField> differentMapping = TypesTests.loadMapping("mapping-numeric.json", true);
assertNotSame(oneMapping, sameMapping); assertNotEquals(basicMapping, numericMapping);
assertEquals(oneMapping, sameMapping);
assertNotEquals(oneMapping, differentMapping);
String wildcard = "*"; String wildcard = "*";
IndexResolution resolution = IndexResolver.merge( IndexResolution resolution = IndexResolver.mergedMapping(wildcard, fromMappings(
Arrays.asList(IndexResolution.valid(new EsIndex("a", oneMapping)), new EsIndex("basic", basicMapping),
IndexResolution.valid(new EsIndex("b", sameMapping)), new EsIndex("numeric", numericMapping)));
IndexResolution.valid(new EsIndex("diff", differentMapping))),
wildcard); assertTrue(resolution.isValid());
assertEquals(basicMapping.size() + numericMapping.size(), resolution.get().mapping().size());
}
public void testMergeIncompatibleTypes() throws Exception {
Map<String, EsField> basicMapping = TypesTests.loadMapping("mapping-basic.json", true);
Map<String, EsField> incompatible = TypesTests.loadMapping("mapping-basic-incompatible.json");
assertNotEquals(basicMapping, incompatible);
String wildcard = "*";
IndexResolution resolution = IndexResolver.mergedMapping(wildcard,
fromMappings(new EsIndex("basic", basicMapping), new EsIndex("incompatible", incompatible)));
assertFalse(resolution.isValid()); assertFalse(resolution.isValid());
MappingException me = expectThrows(MappingException.class, () -> resolution.get());
MappingException ex = expectThrows(MappingException.class, () -> resolution.get());
assertEquals( assertEquals(
"[*] points to indices [a] and [diff] which have different mappings. " "[*] points to indices with incompatible mappings;"
+ "When using multiple indices, the mappings must be identical.", + " field [gender] is mapped in [2] different ways: [text] in [incompatible], [keyword] in [basic]",
ex.getMessage()); me.getMessage());
}
public void testMergeIncompatibleCapabilities() throws Exception {
Map<String, EsField> basicMapping = TypesTests.loadMapping("mapping-basic.json", true);
Map<String, EsField> incompatible = TypesTests.loadMapping("mapping-basic-nodocvalues.json", true);
assertNotEquals(basicMapping, incompatible);
String wildcard = "*";
IndexResolution resolution = IndexResolver.mergedMapping(wildcard,
fromMappings(new EsIndex("basic", basicMapping), new EsIndex("incompatible", incompatible)));
assertFalse(resolution.isValid());
MappingException me = expectThrows(MappingException.class, () -> resolution.get());
assertEquals(
"[*] points to indices with incompatible mappings: field [emp_no] is aggregateable except in [incompatible]",
me.getMessage());
}
public void testMultiLevelObjectMappings() throws Exception {
Map<String, EsField> dottedMapping = TypesTests.loadMapping("mapping-dotted-field.json", true);
String wildcard = "*";
IndexResolution resolution = IndexResolver.mergedMapping(wildcard, fromMappings(new EsIndex("a", dottedMapping)));
assertTrue(resolution.isValid());
assertEqualsMaps(dottedMapping, resolution.get().mapping());
}
public void testMultiLevelNestedMappings() throws Exception {
Map<String, EsField> nestedMapping = TypesTests.loadMapping("mapping-nested.json", true);
String wildcard = "*";
IndexResolution resolution = IndexResolver.mergedMapping(wildcard, fromMappings(new EsIndex("a", nestedMapping)));
assertTrue(resolution.isValid());
assertEqualsMaps(nestedMapping, resolution.get().mapping());
}
private static Map<String, Map<String, FieldCapabilities>> fromMappings(EsIndex... indices) {
Map<String, Map<String, FieldCapabilities>> merged = new HashMap<>();
// first pass: create the field caps
for (EsIndex index : indices) {
for (EsField field : index.mapping().values()) {
addFieldCaps(null, field, index.name(), merged);
}
}
// second pass: update indices
for (Entry<String, Map<String, FieldCapabilities>> entry : merged.entrySet()) {
String fieldName = entry.getKey();
Map<String, FieldCapabilities> caps = entry.getValue();
if (entry.getValue().size() > 1) {
for (EsIndex index : indices) {
EsField field = index.mapping().get(fieldName);
UpdateableFieldCapabilities fieldCaps = (UpdateableFieldCapabilities) caps.get(field.getDataType().esType);
fieldCaps.indices.add(index.name());
}
//TODO: what about nonAgg/SearchIndices?
}
}
return merged;
}
private static void addFieldCaps(String parent, EsField field, String indexName, Map<String, Map<String, FieldCapabilities>> merged) {
String fieldName = parent != null ? parent + "." + field.getName() : field.getName();
Map<String, FieldCapabilities> map = merged.get(fieldName);
if (map == null) {
map = new HashMap<>();
merged.put(fieldName, map);
}
FieldCapabilities caps = map.computeIfAbsent(field.getDataType().esType,
esType -> new UpdateableFieldCapabilities(fieldName, esType,
isSearchable(field.getDataType()),
isAggregatable(field.getDataType())));
if (!field.isAggregatable()) {
((UpdateableFieldCapabilities) caps).nonAggregatableIndices.add(indexName);
}
for (EsField nested : field.getProperties().values()) {
addFieldCaps(fieldName, nested, indexName, merged);
}
}
private static boolean isSearchable(DataType type) {
return type.isPrimitive();
}
private static boolean isAggregatable(DataType type) {
return type.isNumeric() || type == DataType.KEYWORD || type == DataType.DATE;
}
private static class UpdateableFieldCapabilities extends FieldCapabilities {
List<String> indices = new ArrayList<>();
List<String> nonSearchableIndices = new ArrayList<>();
List<String> nonAggregatableIndices = new ArrayList<>();
UpdateableFieldCapabilities(String name, String type, boolean isSearchable, boolean isAggregatable) {
super(name, type, isSearchable, isAggregatable);
}
@Override
public String[] indices() {
return indices.isEmpty() ? null : indices.toArray(new String[indices.size()]);
}
@Override
public String[] nonSearchableIndices() {
return nonSearchableIndices.isEmpty() ? null : nonSearchableIndices.toArray(new String[nonSearchableIndices.size()]);
}
@Override
public String[] nonAggregatableIndices() {
return nonAggregatableIndices.isEmpty() ? null : nonAggregatableIndices.toArray(new String[nonAggregatableIndices.size()]);
}
@Override
public String toString() {
return String.format(Locale.ROOT, "%s,%s->%s", getName(), getType(), indices);
}
}
private static <K, V> void assertEqualsMaps(Map<K, V> left, Map<K, V> right) {
for (Entry<K, V> entry : left.entrySet()) {
V rv = right.get(entry.getKey());
assertEquals(String.format(Locale.ROOT, "Key [%s] has different values", entry.getKey()), entry.getValue(), rv);
}
} }
} }

14
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java
View File

@ -59,10 +59,10 @@ public class TypesTests extends ESTestCase {
assertThat(mapping.size(), is(1)); assertThat(mapping.size(), is(1));
EsField type = mapping.get("full_name"); EsField type = mapping.get("full_name");
assertThat(type, instanceOf(TextEsField.class)); assertThat(type, instanceOf(TextEsField.class));
assertThat(type.hasDocValues(), is(false)); assertThat(type.isAggregatable(), is(false));
TextEsField ttype = (TextEsField) type; TextEsField ttype = (TextEsField) type;
assertThat(type.getPrecision(), is(Integer.MAX_VALUE)); assertThat(type.getPrecision(), is(Integer.MAX_VALUE));
assertThat(ttype.hasDocValues(), is(false)); assertThat(ttype.isAggregatable(), is(false));
} }
public void testKeywordField() { public void testKeywordField() {
@ -71,7 +71,7 @@ public class TypesTests extends ESTestCase {
assertThat(mapping.size(), is(1)); assertThat(mapping.size(), is(1));
EsField field = mapping.get("full_name"); EsField field = mapping.get("full_name");
assertThat(field, instanceOf(KeywordEsField.class)); assertThat(field, instanceOf(KeywordEsField.class));
assertThat(field.hasDocValues(), is(true)); assertThat(field.isAggregatable(), is(true));
assertThat(field.getPrecision(), is(256)); assertThat(field.getPrecision(), is(256));
} }
@ -81,7 +81,7 @@ public class TypesTests extends ESTestCase {
assertThat(mapping.size(), is(1)); assertThat(mapping.size(), is(1));
EsField field = mapping.get("date"); EsField field = mapping.get("date");
assertThat(field.getDataType(), is(DATE)); assertThat(field.getDataType(), is(DATE));
assertThat(field.hasDocValues(), is(true)); assertThat(field.isAggregatable(), is(true));
assertThat(field.getPrecision(), is(24)); assertThat(field.getPrecision(), is(24));
DateEsField dfield = (DateEsField) field; DateEsField dfield = (DateEsField) field;
@ -95,7 +95,7 @@ public class TypesTests extends ESTestCase {
assertThat(mapping.size(), is(1)); assertThat(mapping.size(), is(1));
EsField field = mapping.get("date"); EsField field = mapping.get("date");
assertThat(field.getDataType(), is(DATE)); assertThat(field.getDataType(), is(DATE));
assertThat(field.hasDocValues(), is(true)); assertThat(field.isAggregatable(), is(true));
DateEsField dfield = (DateEsField) field; DateEsField dfield = (DateEsField) field;
// default types // default types
assertThat(dfield.getFormats(), hasSize(2)); assertThat(dfield.getFormats(), hasSize(2));
@ -107,7 +107,7 @@ public class TypesTests extends ESTestCase {
assertThat(mapping.size(), is(1)); assertThat(mapping.size(), is(1));
EsField field = mapping.get("date"); EsField field = mapping.get("date");
assertThat(field.getDataType(), is(DATE)); assertThat(field.getDataType(), is(DATE));
assertThat(field.hasDocValues(), is(true)); assertThat(field.isAggregatable(), is(true));
DateEsField dfield = (DateEsField) field; DateEsField dfield = (DateEsField) field;
// default types // default types
assertThat(dfield.getFormats(), hasSize(1)); assertThat(dfield.getFormats(), hasSize(1));
@ -120,7 +120,7 @@ public class TypesTests extends ESTestCase {
EsField field = mapping.get("session_id"); EsField field = mapping.get("session_id");
assertThat(field, instanceOf(KeywordEsField.class)); assertThat(field, instanceOf(KeywordEsField.class));
assertThat(field.getPrecision(), is(15)); assertThat(field.getPrecision(), is(15));
assertThat(field.hasDocValues(), is(false)); assertThat(field.isAggregatable(), is(false));
} }
public void testDottedField() { public void testDottedField() {

22
x-pack/plugin/sql/src/test/resources/mapping-basic-incompatible.json Normal file
View File

@ -0,0 +1,22 @@
{
"properties" : {
"emp_no" : {
"type" : "long"
},
"first_name" : {
"type" : "text"
},
"gender" : {
"type" : "text"
},
"languages" : {
"type" : "byte"
},
"last_name" : {
"type" : "text"
},
"salary" : {
"type" : "integer"
}
}
}

23
x-pack/plugin/sql/src/test/resources/mapping-basic-nodocvalues.json Normal file
View File

@ -0,0 +1,23 @@
{
"properties" : {
"emp_no" : {
"type" : "integer",
"doc_values" : false
},
"first_name" : {
"type" : "text"
},
"gender" : {
"type" : "keyword"
},
"languages" : {
"type" : "byte"
},
"last_name" : {
"type" : "text"
},
"salary" : {
"type" : "integer"
}
}
}

10
x-pack/qa/sql/security/roles.yml
View File

@ -7,8 +7,8 @@ rest_minimal:
privileges: [read, "indices:admin/get"] privileges: [read, "indices:admin/get"]
# end::rest # end::rest
# tag::cli_jdbc # tag::cli_drivers
cli_or_jdbc_minimal: cli_or_drivers_minimal:
cluster: cluster:
- "cluster:monitor/main" - "cluster:monitor/main"
indices: indices:
@ -16,7 +16,7 @@ cli_or_jdbc_minimal:
privileges: [read, "indices:admin/get"] privileges: [read, "indices:admin/get"]
- names: bort - names: bort
privileges: [read, "indices:admin/get"] privileges: [read, "indices:admin/get"]
# end::cli_jdbc # end::cli_drivers
read_something_else: read_something_else:
cluster: cluster:
@ -82,6 +82,6 @@ no_get_index:
- "cluster:monitor/main" - "cluster:monitor/main"
indices: indices:
- names: test - names: test
privileges: [read] privileges: [monitor]
- names: bort - names: bort
privileges: [read] privileges: [monitor]

23
x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliSecurityIT.java
View File

@ -7,9 +7,10 @@ package org.elasticsearch.xpack.qa.sql.security;
import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.xpack.qa.sql.cli.ErrorsTestCase;
import org.elasticsearch.xpack.qa.sql.cli.EmbeddedCli; import org.elasticsearch.xpack.qa.sql.cli.EmbeddedCli;
import org.elasticsearch.xpack.qa.sql.cli.EmbeddedCli.SecurityConfig; import org.elasticsearch.xpack.qa.sql.cli.EmbeddedCli.SecurityConfig;
import org.elasticsearch.xpack.qa.sql.cli.ErrorsTestCase;
import java.io.IOException; import java.io.IOException;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.nio.file.Files; import java.nio.file.Files;
@ -20,7 +21,6 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import static org.elasticsearch.xpack.qa.sql.cli.CliIntegrationTestCase.elasticsearchAddress; import static org.elasticsearch.xpack.qa.sql.cli.CliIntegrationTestCase.elasticsearchAddress;
import static org.hamcrest.Matchers.both;
import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.startsWith; import static org.hamcrest.Matchers.startsWith;
@ -53,7 +53,7 @@ public class CliSecurityIT extends SqlSecurityTestCase {
private static class CliActions implements Actions { private static class CliActions implements Actions {
@Override @Override
public String minimalPermissionsForAllActions() { public String minimalPermissionsForAllActions() {
return "cli_or_jdbc_minimal"; return "cli_or_drivers_minimal";
} }
private SecurityConfig userSecurity(String user) { private SecurityConfig userSecurity(String user) {
@ -121,12 +121,19 @@ public class CliSecurityIT extends SqlSecurityTestCase {
} }
@Override @Override
public void expectDescribe(Map<String, String> columns, String user) throws Exception { public void expectDescribe(Map<String, List<String>> columns, String user) throws Exception {
try (EmbeddedCli cli = new EmbeddedCli(elasticsearchAddress(), true, userSecurity(user))) { try (EmbeddedCli cli = new EmbeddedCli(elasticsearchAddress(), true, userSecurity(user))) {
assertThat(cli.command("DESCRIBE test"), containsString("column | type")); String output = cli.command("DESCRIBE test");
assertEquals("---------------+---------------", cli.readLine()); assertThat(output, containsString("column"));
for (Map.Entry<String, String> column : columns.entrySet()) { assertThat(output, containsString("type"));
assertThat(cli.readLine(), both(startsWith(column.getKey())).and(containsString("|" + column.getValue()))); assertThat(output, containsString("mapping"));
assertThat(cli.readLine(), containsString("-+---------------+---------------"));
for (Map.Entry<String, List<String>> column : columns.entrySet()) {
String line = cli.readLine();
assertThat(line, startsWith(column.getKey()));
for (String value : column.getValue()) {
assertThat(line, containsString(value));
}
} }
assertEquals("", cli.readLine()); assertEquals("", cli.readLine());
} }

20
x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcSecurityIT.java
View File

@ -118,7 +118,7 @@ public class JdbcSecurityIT extends SqlSecurityTestCase {
private static class JdbcActions implements Actions { private static class JdbcActions implements Actions {
@Override @Override
public String minimalPermissionsForAllActions() { public String minimalPermissionsForAllActions() {
return "cli_or_jdbc_minimal"; return "cli_or_drivers_minimal";
} }
@Override @Override
@ -158,22 +158,26 @@ public class JdbcSecurityIT extends SqlSecurityTestCase {
} }
@Override @Override
public void expectDescribe(Map<String, String> columns, String user) throws Exception { public void expectDescribe(Map<String, List<String>> columns, String user) throws Exception {
try (Connection h2 = LocalH2.anonymousDb(); try (Connection h2 = LocalH2.anonymousDb();
Connection es = es(userProperties(user))) { Connection es = es(userProperties(user))) {
// h2 doesn't have the same sort of DESCRIBE that we have so we emulate it // h2 doesn't have the same sort of DESCRIBE that we have so we emulate it
h2.createStatement().executeUpdate("CREATE TABLE mock (column VARCHAR, type VARCHAR)"); h2.createStatement().executeUpdate("CREATE TABLE mock (column VARCHAR, type VARCHAR, mapping VARCHAR)");
if (columns.size() > 0) { if (columns.size() > 0) {
StringBuilder insert = new StringBuilder(); StringBuilder insert = new StringBuilder();
insert.append("INSERT INTO mock (column, type) VALUES "); insert.append("INSERT INTO mock (column, type, mapping) VALUES ");
boolean first = true; boolean first = true;
for (Map.Entry<String, String> column : columns.entrySet()) { for (Map.Entry<String, List<String>> column : columns.entrySet()) {
if (first) { if (first) {
first = false; first = false;
} else { } else {
insert.append(", "); insert.append(", ");
} }
insert.append("('").append(column.getKey()).append("', '").append(column.getValue()).append("')"); insert.append("('").append(column.getKey()).append("'");
for (String value : column.getValue()) {
insert.append(", '").append(value).append("'");
}
insert.append(")");
} }
h2.createStatement().executeUpdate(insert.toString()); h2.createStatement().executeUpdate(insert.toString());
} }
@ -250,7 +254,7 @@ public class JdbcSecurityIT extends SqlSecurityTestCase {
// Metadata methods only available to JDBC // Metadata methods only available to JDBC
public void testMetaDataGetTablesWithFullAccess() throws Exception { public void testMetaDataGetTablesWithFullAccess() throws Exception {
createUser("full_access", "cli_or_jdbc_minimal"); createUser("full_access", "cli_or_drivers_minimal");
expectActionMatchesAdmin( expectActionMatchesAdmin(
con -> con.getMetaData().getTables("%", "%", "%t", null), con -> con.getMetaData().getTables("%", "%", "%t", null),
@ -283,7 +287,7 @@ public class JdbcSecurityIT extends SqlSecurityTestCase {
} }
public void testMetaDataGetColumnsWorksAsFullAccess() throws Exception { public void testMetaDataGetColumnsWorksAsFullAccess() throws Exception {
createUser("full_access", "cli_or_jdbc_minimal"); createUser("full_access", "cli_or_drivers_minimal");
expectActionMatchesAdmin( expectActionMatchesAdmin(
con -> con.getMetaData().getColumns(null, "%", "%t", "%"), con -> con.getMetaData().getColumns(null, "%", "%t", "%"),

14
x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java
View File

@ -93,15 +93,19 @@ public class RestSqlSecurityIT extends SqlSecurityTestCase {
} }
@Override @Override
public void expectDescribe(Map<String, String> columns, String user) throws Exception { public void expectDescribe(Map<String, List<String>> columns, String user) throws Exception {
String mode = randomMode(); String mode = randomMode();
Map<String, Object> expected = new HashMap<>(3); Map<String, Object> expected = new HashMap<>(3);
expected.put("columns", Arrays.asList( expected.put("columns", Arrays.asList(
columnInfo(mode, "column", "keyword", JDBCType.VARCHAR, 0), columnInfo(mode, "column", "keyword", JDBCType.VARCHAR, 0),
columnInfo(mode, "type", "keyword", JDBCType.VARCHAR, 0))); columnInfo(mode, "type", "keyword", JDBCType.VARCHAR, 0),
columnInfo(mode, "mapping", "keyword", JDBCType.VARCHAR, 0)));
List<List<String>> rows = new ArrayList<>(columns.size()); List<List<String>> rows = new ArrayList<>(columns.size());
for (Map.Entry<String, String> column : columns.entrySet()) { for (Map.Entry<String, List<String>> column : columns.entrySet()) {
rows.add(Arrays.asList(column.getKey(), column.getValue())); List<String> cols = new ArrayList<>();
cols.add(column.getKey());
cols.addAll(column.getValue());
rows.add(cols);
} }
expected.put("rows", rows); expected.put("rows", rows);
@ -232,7 +236,7 @@ public class RestSqlSecurityIT extends SqlSecurityTestCase {
assertEquals(404, e.getResponse().getStatusLine().getStatusCode()); assertEquals(404, e.getResponse().getStatusLine().getStatusCode());
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("test_admin", "test") .expectSqlCompositeActionFieldCaps("test_admin", "test")
.expect(true, SQL_ACTION_NAME, "full_access", empty()) .expect(true, SQL_ACTION_NAME, "full_access", empty())
// one scroll access denied per shard // one scroll access denied per shard
.expect("access_denied", SQL_ACTION_NAME, "full_access", "default_native", empty(), "InternalScrollSearchRequest") .expect("access_denied", SQL_ACTION_NAME, "full_access", "default_native", empty(), "InternalScrollSearchRequest")

101
x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java
View File

@ -10,6 +10,8 @@ import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.SpecialPermission; import org.elasticsearch.SpecialPermission;
import org.elasticsearch.action.admin.indices.get.GetIndexAction; import org.elasticsearch.action.admin.indices.get.GetIndexAction;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction;
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
import org.elasticsearch.client.Request; import org.elasticsearch.client.Request;
import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.ResponseException;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
@ -40,11 +42,12 @@ import java.util.Map;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.function.Function; import java.util.function.Function;
import static java.util.Arrays.asList;
import static java.util.Collections.singletonMap; import static java.util.Collections.singletonMap;
import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.hasItems;
import static org.hamcrest.Matchers.is;
public abstract class SqlSecurityTestCase extends ESRestTestCase { public abstract class SqlSecurityTestCase extends ESRestTestCase {
/** /**
@ -65,7 +68,7 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
* to 1 and completely scrolls the results. * to 1 and completely scrolls the results.
*/ */
void expectScrollMatchesAdmin(String adminSql, String user, String userSql) throws Exception; void expectScrollMatchesAdmin(String adminSql, String user, String userSql) throws Exception;
void expectDescribe(Map<String, String> columns, String user) throws Exception; void expectDescribe(Map<String, List<String>> columns, String user) throws Exception;
void expectShowTables(List<String> tables, String user) throws Exception; void expectShowTables(List<String> tables, String user) throws Exception;
void expectForbidden(String user, String sql) throws Exception; void expectForbidden(String user, String sql) throws Exception;
void expectUnknownIndex(String user, String sql) throws Exception; void expectUnknownIndex(String user, String sql) throws Exception;
@ -196,7 +199,7 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
public void testQueryWorksAsAdmin() throws Exception { public void testQueryWorksAsAdmin() throws Exception {
actions.queryWorksAsAdmin(); actions.queryWorksAsAdmin();
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("test_admin", "test") .expectSqlCompositeActionFieldCaps("test_admin", "test")
.assertLogs(); .assertLogs();
} }
@ -205,8 +208,8 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
actions.expectMatchesAdmin("SELECT * FROM test ORDER BY a", "full_access", "SELECT * FROM test ORDER BY a"); actions.expectMatchesAdmin("SELECT * FROM test ORDER BY a", "full_access", "SELECT * FROM test ORDER BY a");
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("test_admin", "test") .expectSqlCompositeActionFieldCaps("test_admin", "test")
.expectSqlCompositeAction("full_access", "test") .expectSqlCompositeActionFieldCaps("full_access", "test")
.assertLogs(); .assertLogs();
} }
@ -215,12 +218,12 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
actions.expectScrollMatchesAdmin("SELECT * FROM test ORDER BY a", "full_access", "SELECT * FROM test ORDER BY a"); actions.expectScrollMatchesAdmin("SELECT * FROM test ORDER BY a", "full_access", "SELECT * FROM test ORDER BY a");
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("test_admin", "test") .expectSqlCompositeActionFieldCaps("test_admin", "test")
/* Scrolling doesn't have to access the index again, at least not through sql. /* Scrolling doesn't have to access the index again, at least not through sql.
* If we asserted query and scroll logs then we would see the scroll. */ * If we asserted query and scroll logs then we would see the scroll. */
.expect(true, SQL_ACTION_NAME, "test_admin", empty()) .expect(true, SQL_ACTION_NAME, "test_admin", empty())
.expect(true, SQL_ACTION_NAME, "test_admin", empty()) .expect(true, SQL_ACTION_NAME, "test_admin", empty())
.expectSqlCompositeAction("full_access", "test") .expectSqlCompositeActionFieldCaps("full_access", "test")
.expect(true, SQL_ACTION_NAME, "full_access", empty()) .expect(true, SQL_ACTION_NAME, "full_access", empty())
.expect(true, SQL_ACTION_NAME, "full_access", empty()) .expect(true, SQL_ACTION_NAME, "full_access", empty())
.assertLogs(); .assertLogs();
@ -243,7 +246,7 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
//This user has permission to run sql queries so they are given preliminary authorization //This user has permission to run sql queries so they are given preliminary authorization
.expect(true, SQL_ACTION_NAME, "wrong_access", empty()) .expect(true, SQL_ACTION_NAME, "wrong_access", empty())
//the following get index is granted too but against the no indices placeholder, as ignore_unavailable=true //the following get index is granted too but against the no indices placeholder, as ignore_unavailable=true
.expect(true, GetIndexAction.NAME, "wrong_access", hasItems("*", "-*")) .expect(true, FieldCapabilitiesAction.NAME, "wrong_access", hasItems("*", "-*"))
.assertLogs(); .assertLogs();
} }
@ -252,8 +255,8 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
actions.expectMatchesAdmin("SELECT a FROM test ORDER BY a", "only_a", "SELECT * FROM test ORDER BY a"); actions.expectMatchesAdmin("SELECT a FROM test ORDER BY a", "only_a", "SELECT * FROM test ORDER BY a");
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("test_admin", "test") .expectSqlCompositeActionFieldCaps("test_admin", "test")
.expectSqlCompositeAction("only_a", "test") .expectSqlCompositeActionFieldCaps("only_a", "test")
.assertLogs(); .assertLogs();
} }
@ -262,18 +265,18 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
actions.expectScrollMatchesAdmin("SELECT a FROM test ORDER BY a", "only_a", "SELECT * FROM test ORDER BY a"); actions.expectScrollMatchesAdmin("SELECT a FROM test ORDER BY a", "only_a", "SELECT * FROM test ORDER BY a");
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("test_admin", "test") .expectSqlCompositeActionFieldCaps("test_admin", "test")
/* Scrolling doesn't have to access the index again, at least not through sql. /* Scrolling doesn't have to access the index again, at least not through sql.
* If we asserted query and scroll logs then we would see the scroll. */ * If we asserted query and scroll logs then we would see the scroll. */
.expect(true, SQL_ACTION_NAME, "test_admin", empty()) .expect(true, SQL_ACTION_NAME, "test_admin", empty())
.expect(true, SQL_ACTION_NAME, "test_admin", empty()) .expect(true, SQL_ACTION_NAME, "test_admin", empty())
.expectSqlCompositeAction("only_a", "test") .expectSqlCompositeActionFieldCaps("only_a", "test")
.expect(true, SQL_ACTION_NAME, "only_a", empty()) .expect(true, SQL_ACTION_NAME, "only_a", empty())
.expect(true, SQL_ACTION_NAME, "only_a", empty()) .expect(true, SQL_ACTION_NAME, "only_a", empty())
.assertLogs(); .assertLogs();
} }
public void testQueryStringSingeFieldGrantedWrongRequested() throws Exception { public void testQueryStringSingleFieldGrantedWrongRequested() throws Exception {
createUser("only_a", "read_test_a"); createUser("only_a", "read_test_a");
actions.expectUnknownColumn("only_a", "SELECT c FROM test", "c"); actions.expectUnknownColumn("only_a", "SELECT c FROM test", "c");
@ -284,7 +287,7 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
* out but it failed in SQL because it couldn't compile the * out but it failed in SQL because it couldn't compile the
* query without the metadata for the missing field. */ * query without the metadata for the missing field. */
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("only_a", "test") .expectSqlCompositeActionFieldCaps("only_a", "test")
.assertLogs(); .assertLogs();
} }
@ -293,8 +296,8 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
actions.expectMatchesAdmin("SELECT a, b FROM test ORDER BY a", "not_c", "SELECT * FROM test ORDER BY a"); actions.expectMatchesAdmin("SELECT a, b FROM test ORDER BY a", "not_c", "SELECT * FROM test ORDER BY a");
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("test_admin", "test") .expectSqlCompositeActionFieldCaps("test_admin", "test")
.expectSqlCompositeAction("not_c", "test") .expectSqlCompositeActionFieldCaps("not_c", "test")
.assertLogs(); .assertLogs();
} }
@ -303,12 +306,12 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
actions.expectScrollMatchesAdmin("SELECT a, b FROM test ORDER BY a", "not_c", "SELECT * FROM test ORDER BY a"); actions.expectScrollMatchesAdmin("SELECT a, b FROM test ORDER BY a", "not_c", "SELECT * FROM test ORDER BY a");
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("test_admin", "test") .expectSqlCompositeActionFieldCaps("test_admin", "test")
/* Scrolling doesn't have to access the index again, at least not through sql. /* Scrolling doesn't have to access the index again, at least not through sql.
* If we asserted query and scroll logs then we would see the scroll. */ * If we asserted query and scroll logs then we would see the scroll. */
.expect(true, SQL_ACTION_NAME, "test_admin", empty()) .expect(true, SQL_ACTION_NAME, "test_admin", empty())
.expect(true, SQL_ACTION_NAME, "test_admin", empty()) .expect(true, SQL_ACTION_NAME, "test_admin", empty())
.expectSqlCompositeAction("not_c", "test") .expectSqlCompositeActionFieldCaps("not_c", "test")
.expect(true, SQL_ACTION_NAME, "not_c", empty()) .expect(true, SQL_ACTION_NAME, "not_c", empty())
.expect(true, SQL_ACTION_NAME, "not_c", empty()) .expect(true, SQL_ACTION_NAME, "not_c", empty())
.assertLogs(); .assertLogs();
@ -325,7 +328,7 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
* out but it failed in SQL because it couldn't compile the * out but it failed in SQL because it couldn't compile the
* query without the metadata for the missing field. */ * query without the metadata for the missing field. */
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("not_c", "test") .expectSqlCompositeActionFieldCaps("not_c", "test")
.assertLogs(); .assertLogs();
} }
@ -334,15 +337,15 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
actions.expectMatchesAdmin("SELECT * FROM test WHERE c != 3 ORDER BY a", "no_3s", "SELECT * FROM test ORDER BY a"); actions.expectMatchesAdmin("SELECT * FROM test WHERE c != 3 ORDER BY a", "no_3s", "SELECT * FROM test ORDER BY a");
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("test_admin", "test") .expectSqlCompositeActionFieldCaps("test_admin", "test")
.expectSqlCompositeAction("no_3s", "test") .expectSqlCompositeActionFieldCaps("no_3s", "test")
.assertLogs(); .assertLogs();
} }
public void testShowTablesWorksAsAdmin() throws Exception { public void testShowTablesWorksAsAdmin() throws Exception {
actions.expectShowTables(Arrays.asList("bort", "test"), null); actions.expectShowTables(Arrays.asList("bort", "test"), null);
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("test_admin", "bort", "test") .expectSqlCompositeActionGetIndex("test_admin", "bort", "test")
.assertLogs(); .assertLogs();
} }
@ -351,8 +354,8 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
actions.expectMatchesAdmin("SHOW TABLES LIKE '%t'", "full_access", "SHOW TABLES"); actions.expectMatchesAdmin("SHOW TABLES LIKE '%t'", "full_access", "SHOW TABLES");
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("test_admin", "bort", "test") .expectSqlCompositeActionGetIndex("test_admin", "bort", "test")
.expectSqlCompositeAction("full_access", "bort", "test") .expectSqlCompositeActionGetIndex("full_access", "bort", "test")
.assertLogs(); .assertLogs();
} }
@ -370,8 +373,7 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
actions.expectMatchesAdmin("SHOW TABLES LIKE 'bort'", "read_bort", "SHOW TABLES"); actions.expectMatchesAdmin("SHOW TABLES LIKE 'bort'", "read_bort", "SHOW TABLES");
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("test_admin", "bort") .expectSqlCompositeActionGetIndex("test_admin", "bort").expectSqlCompositeActionGetIndex("read_bort", "bort")
.expectSqlCompositeAction("read_bort", "bort")
.assertLogs(); .assertLogs();
} }
@ -388,13 +390,13 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
} }
public void testDescribeWorksAsAdmin() throws Exception { public void testDescribeWorksAsAdmin() throws Exception {
Map<String, String> expected = new TreeMap<>(); Map<String, List<String>> expected = new TreeMap<>();
expected.put("a", "BIGINT"); expected.put("a", asList("BIGINT", "LONG"));
expected.put("b", "BIGINT"); expected.put("b", asList("BIGINT", "LONG"));
expected.put("c", "BIGINT"); expected.put("c", asList("BIGINT", "LONG"));
actions.expectDescribe(expected, null); actions.expectDescribe(expected, null);
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("test_admin", "test") .expectSqlCompositeActionFieldCaps("test_admin", "test")
.assertLogs(); .assertLogs();
} }
@ -403,8 +405,8 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
actions.expectMatchesAdmin("DESCRIBE test", "full_access", "DESCRIBE test"); actions.expectMatchesAdmin("DESCRIBE test", "full_access", "DESCRIBE test");
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("test_admin", "test") .expectSqlCompositeActionFieldCaps("test_admin", "test")
.expectSqlCompositeAction("full_access", "test") .expectSqlCompositeActionFieldCaps("full_access", "test")
.assertLogs(); .assertLogs();
} }
@ -425,28 +427,28 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
//This user has permission to run sql queries so they are given preliminary authorization //This user has permission to run sql queries so they are given preliminary authorization
.expect(true, SQL_ACTION_NAME, "wrong_access", empty()) .expect(true, SQL_ACTION_NAME, "wrong_access", empty())
//the following get index is granted too but against the no indices placeholder, as ignore_unavailable=true //the following get index is granted too but against the no indices placeholder, as ignore_unavailable=true
.expect(true, GetIndexAction.NAME, "wrong_access", hasItems("*", "-*")) .expect(true, FieldCapabilitiesAction.NAME, "wrong_access", hasItems("*", "-*"))
.assertLogs(); .assertLogs();
} }
public void testDescribeSingleFieldGranted() throws Exception { public void testDescribeSingleFieldGranted() throws Exception {
createUser("only_a", "read_test_a"); createUser("only_a", "read_test_a");
actions.expectDescribe(singletonMap("a", "BIGINT"), "only_a"); actions.expectDescribe(singletonMap("a", asList("BIGINT", "LONG")), "only_a");
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("only_a", "test") .expectSqlCompositeActionFieldCaps("only_a", "test")
.assertLogs(); .assertLogs();
} }
public void testDescribeSingleFieldExcepted() throws Exception { public void testDescribeSingleFieldExcepted() throws Exception {
createUser("not_c", "read_test_a_and_b"); createUser("not_c", "read_test_a_and_b");
Map<String, String> expected = new TreeMap<>(); Map<String, List<String>> expected = new TreeMap<>();
expected.put("a", "BIGINT"); expected.put("a", asList("BIGINT", "LONG"));
expected.put("b", "BIGINT"); expected.put("b", asList("BIGINT", "LONG"));
actions.expectDescribe(expected, "not_c"); actions.expectDescribe(expected, "not_c");
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("not_c", "test") .expectSqlCompositeActionFieldCaps("not_c", "test")
.assertLogs(); .assertLogs();
} }
@ -455,8 +457,8 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
actions.expectMatchesAdmin("DESCRIBE test", "no_3s", "DESCRIBE test"); actions.expectMatchesAdmin("DESCRIBE test", "no_3s", "DESCRIBE test");
createAuditLogAsserter() createAuditLogAsserter()
.expectSqlCompositeAction("test_admin", "test") .expectSqlCompositeActionFieldCaps("test_admin", "test")
.expectSqlCompositeAction("no_3s", "test") .expectSqlCompositeActionFieldCaps("no_3s", "test")
.assertLogs(); .assertLogs();
} }
@ -497,12 +499,18 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
protected class AuditLogAsserter { protected class AuditLogAsserter {
protected final List<Function<Map<String, Object>, Boolean>> logCheckers = new ArrayList<>(); protected final List<Function<Map<String, Object>, Boolean>> logCheckers = new ArrayList<>();
public AuditLogAsserter expectSqlCompositeAction(String user, String... indices) { public AuditLogAsserter expectSqlCompositeActionGetIndex(String user, String... indices) {
expect(true, SQL_ACTION_NAME, user, empty()); expect(true, SQL_ACTION_NAME, user, empty());
expect(true, GetIndexAction.NAME, user, hasItems(indices)); expect(true, GetIndexAction.NAME, user, hasItems(indices));
return this; return this;
} }
public AuditLogAsserter expectSqlCompositeActionFieldCaps(String user, String... indices) {
expect(true, SQL_ACTION_NAME, user, empty());
expect(true, FieldCapabilitiesAction.NAME, user, hasItems(indices));
return this;
}
public AuditLogAsserter expect(boolean granted, String action, String principal, public AuditLogAsserter expect(boolean granted, String action, String principal,
Matcher<? extends Iterable<? extends String>> indicesMatcher) { Matcher<? extends Iterable<? extends String>> indicesMatcher) {
String request; String request;
@ -513,6 +521,9 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
case GetIndexAction.NAME: case GetIndexAction.NAME:
request = GetIndexRequest.class.getSimpleName(); request = GetIndexRequest.class.getSimpleName();
break; break;
case FieldCapabilitiesAction.NAME:
request = FieldCapabilitiesRequest.class.getSimpleName();
break;
default: default:
throw new IllegalArgumentException("Unknown action [" + action + "]"); throw new IllegalArgumentException("Unknown action [" + action + "]");
} }
@ -564,7 +575,9 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
continue; continue;
} }
assertThat(log.containsKey("action"), is(true)); assertThat(log.containsKey("action"), is(true));
if (false == (SQL_ACTION_NAME.equals(log.get("action")) || GetIndexAction.NAME.equals(log.get("action")))) { if (false == (SQL_ACTION_NAME.equals(log.get("action"))
|| GetIndexAction.NAME.equals(log.get("action"))
|| FieldCapabilitiesAction.NAME.equals(log.get("action")))) {
// TODO we may want to extend this and the assertions to SearchAction.NAME as well // TODO we may want to extend this and the assertions to SearchAction.NAME as well
continue; continue;
} }

10
x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/EmbeddedCli.java
View File

@ -5,13 +5,13 @@
*/ */
package org.elasticsearch.xpack.qa.sql.cli; package org.elasticsearch.xpack.qa.sql.cli;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cli.MockTerminal;
import org.elasticsearch.cli.Terminal; import org.elasticsearch.cli.Terminal;
import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.xpack.sql.cli.Cli; import org.elasticsearch.xpack.sql.cli.Cli;
import org.elasticsearch.xpack.sql.cli.CliTerminal; import org.elasticsearch.xpack.sql.cli.CliTerminal;
import org.elasticsearch.xpack.sql.cli.JLineTerminal; import org.elasticsearch.xpack.sql.cli.JLineTerminal;
@ -49,7 +49,7 @@ import static org.junit.Assert.fail;
* and doesn't run super frequently. * and doesn't run super frequently.
*/ */
public class EmbeddedCli implements Closeable { public class EmbeddedCli implements Closeable {
private static final Logger logger = Loggers.getLogger(EmbeddedCli.class); private static final Logger logger = LogManager.getLogger(EmbeddedCli.class);
private final Thread exec; private final Thread exec;
private final Cli cli; private final Cli cli;
@ -151,7 +151,9 @@ public class EmbeddedCli implements Closeable {
} }
// Throw out the logo // Throw out the logo
while (false == readLine().contains("SQL")); while (false == readLine().contains("SQL")) {
;
}
assertConnectionTest(); assertConnectionTest();
} catch (IOException e) { } catch (IOException e) {
try { try {

6
x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java
View File

@ -5,9 +5,10 @@
*/ */
package org.elasticsearch.xpack.qa.sql.cli; package org.elasticsearch.xpack.qa.sql.cli;
import java.io.IOException;
import org.elasticsearch.client.Request; import org.elasticsearch.client.Request;
import java.io.IOException;
import static org.hamcrest.Matchers.startsWith; import static org.hamcrest.Matchers.startsWith;
/** /**
@ -43,7 +44,8 @@ public abstract class ErrorsTestCase extends CliIntegrationTestCase implements o
client().performRequest(request); client().performRequest(request);
assertFoundOneProblem(command("SELECT * FROM test")); assertFoundOneProblem(command("SELECT * FROM test"));
assertEquals("line 1:15: [test] doesn't have any types so it is incompatible with sql" + END, readLine()); //assertEquals("line 1:15: [test] doesn't have any types so it is incompatible with sql" + END, readLine());
assertEquals("line 1:15: Unknown index [test]" + END, readLine());
} }
@Override @Override

30
x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DataLoader.java
View File

@ -43,7 +43,7 @@ public class DataLoader {
protected static void loadEmpDatasetIntoEs(RestClient client) throws Exception { protected static void loadEmpDatasetIntoEs(RestClient client) throws Exception {
loadEmpDatasetIntoEs(client, "test_emp", "employees"); loadEmpDatasetIntoEs(client, "test_emp", "employees");
loadEmpDatasetIntoEs(client, "test_emp_copy", "employees"); loadEmpDatasetWithExtraIntoEs(client, "test_emp_copy", "employees");
makeAlias(client, "test_alias", "test_emp", "test_emp_copy"); makeAlias(client, "test_alias", "test_emp", "test_emp_copy");
makeAlias(client, "test_alias_emp", "test_emp", "test_emp_copy"); makeAlias(client, "test_alias_emp", "test_emp", "test_emp_copy");
} }
@ -63,6 +63,14 @@ public class DataLoader {
} }
protected static void loadEmpDatasetIntoEs(RestClient client, String index, String fileName) throws Exception { protected static void loadEmpDatasetIntoEs(RestClient client, String index, String fileName) throws Exception {
loadEmpDatasetIntoEs(client, index, fileName, false);
}
protected static void loadEmpDatasetWithExtraIntoEs(RestClient client, String index, String fileName) throws Exception {
loadEmpDatasetIntoEs(client, index, fileName, true);
}
private static void loadEmpDatasetIntoEs(RestClient client, String index, String fileName, boolean extraFields) throws Exception {
Request request = new Request("PUT", "/" + index); Request request = new Request("PUT", "/" + index);
XContentBuilder createIndex = JsonXContent.contentBuilder().startObject(); XContentBuilder createIndex = JsonXContent.contentBuilder().startObject();
createIndex.startObject("settings"); createIndex.startObject("settings");
@ -76,10 +84,26 @@ public class DataLoader {
{ {
createIndex.startObject("properties"); createIndex.startObject("properties");
{ {
createIndex.startObject("emp_no").field("type", "integer").endObject(); createIndex.startObject("emp_no").field("type", "integer");
if (extraFields) {
createIndex.field("copy_to", "extra_no");
}
createIndex.endObject();
if (extraFields) {
createIndex.startObject("extra_no").field("type", "integer").endObject();
}
createString("first_name", createIndex); createString("first_name", createIndex);
createString("last_name", createIndex); createString("last_name", createIndex);
createIndex.startObject("gender").field("type", "keyword").endObject(); createIndex.startObject("gender").field("type", "keyword");
if (extraFields) {
createIndex.field("copy_to", "extra_gender");
}
createIndex.endObject();
if (extraFields) {
createIndex.startObject("extra_gender").field("type", "keyword").endObject();
}
createIndex.startObject("birth_date").field("type", "date").endObject(); createIndex.startObject("birth_date").field("type", "date").endObject();
createIndex.startObject("hire_date").field("type", "date").endObject(); createIndex.startObject("hire_date").field("type", "date").endObject();
createIndex.startObject("salary").field("type", "integer").endObject(); createIndex.startObject("salary").field("type", "integer").endObject();

7
x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java
View File

@ -5,9 +5,10 @@
*/ */
package org.elasticsearch.xpack.qa.sql.jdbc; package org.elasticsearch.xpack.qa.sql.jdbc;
import org.elasticsearch.client.Request;
import java.sql.Connection; import java.sql.Connection;
import java.sql.SQLException; import java.sql.SQLException;
import org.elasticsearch.client.Request;
import static org.hamcrest.Matchers.startsWith; import static org.hamcrest.Matchers.startsWith;
@ -40,7 +41,9 @@ public class ErrorsTestCase extends JdbcIntegrationTestCase implements org.elast
try (Connection c = esJdbc()) { try (Connection c = esJdbc()) {
SQLException e = expectThrows(SQLException.class, () -> c.prepareStatement("SELECT * FROM test").executeQuery()); SQLException e = expectThrows(SQLException.class, () -> c.prepareStatement("SELECT * FROM test").executeQuery());
assertEquals("Found 1 problem(s)\nline 1:15: [test] doesn't have any types so it is incompatible with sql", e.getMessage()); // see https://github.com/elastic/elasticsearch/issues/34719
//assertEquals("Found 1 problem(s)\nline 1:15: [test] doesn't have any types so it is incompatible with sql", e.getMessage());
assertEquals("Found 1 problem(s)\nline 1:15: Unknown index [test]", e.getMessage());
} }
} }

4
x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java
View File

@ -219,7 +219,9 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
client().performRequest(request); client().performRequest(request);
String mode = randomFrom("jdbc", "plain"); String mode = randomFrom("jdbc", "plain");
expectBadRequest(() -> runSql(mode, "SELECT * FROM test"), expectBadRequest(() -> runSql(mode, "SELECT * FROM test"),
containsString("1:15: [test] doesn't have any types so it is incompatible with sql")); // see https://github.com/elastic/elasticsearch/issues/34719
//containsString("1:15: [test] doesn't have any types so it is incompatible with sql"));
containsString("1:15: Unknown index [test]"));
} }
@Override @Override

76
x-pack/qa/sql/src/main/resources/alias.csv-spec
View File

@ -26,47 +26,51 @@ emp_no:i | first_name:s
describeAlias describeAlias
DESCRIBE test_alias; DESCRIBE test_alias;
column:s | type:s column | type | mapping
--------------------+---------------+---------------
birth_date | TIMESTAMP birth_date |TIMESTAMP |DATE
dep | STRUCT dep |STRUCT |NESTED
dep.dep_id | VARCHAR dep.dep_id |VARCHAR |KEYWORD
dep.dep_name | VARCHAR dep.dep_name |VARCHAR |TEXT
dep.dep_name.keyword | VARCHAR dep.dep_name.keyword|VARCHAR |KEYWORD
dep.from_date | TIMESTAMP dep.from_date |TIMESTAMP |DATE
dep.to_date | TIMESTAMP dep.to_date |TIMESTAMP |DATE
emp_no | INTEGER emp_no |INTEGER |INTEGER
first_name | VARCHAR extra_gender |VARCHAR |KEYWORD
first_name.keyword | VARCHAR extra_no |INTEGER |INTEGER
gender | VARCHAR first_name |VARCHAR |TEXT
hire_date | TIMESTAMP first_name.keyword |VARCHAR |KEYWORD
languages | TINYINT gender |VARCHAR |KEYWORD
last_name | VARCHAR hire_date |TIMESTAMP |DATE
last_name.keyword | VARCHAR languages |TINYINT |BYTE
salary | INTEGER last_name |VARCHAR |TEXT
last_name.keyword |VARCHAR |KEYWORD
salary |INTEGER |INTEGER
; ;
describePattern describePattern
DESCRIBE "test_*"; DESCRIBE "test_*";
column:s | type:s column | type | mapping
--------------------+---------------+---------------
birth_date | TIMESTAMP birth_date |TIMESTAMP |DATE
dep | STRUCT dep |STRUCT |NESTED
dep.dep_id | VARCHAR dep.dep_id |VARCHAR |KEYWORD
dep.dep_name | VARCHAR dep.dep_name |VARCHAR |TEXT
dep.dep_name.keyword | VARCHAR dep.dep_name.keyword|VARCHAR |KEYWORD
dep.from_date | TIMESTAMP dep.from_date |TIMESTAMP |DATE
dep.to_date | TIMESTAMP dep.to_date |TIMESTAMP |DATE
emp_no | INTEGER emp_no |INTEGER |INTEGER
first_name | VARCHAR extra_gender |VARCHAR |KEYWORD
first_name.keyword | VARCHAR extra_no |INTEGER |INTEGER
gender | VARCHAR first_name |VARCHAR |TEXT
hire_date | TIMESTAMP first_name.keyword |VARCHAR |KEYWORD
languages | TINYINT gender |VARCHAR |KEYWORD
last_name | VARCHAR hire_date |TIMESTAMP |DATE
last_name.keyword | VARCHAR languages |TINYINT |BYTE
salary | INTEGER last_name |VARCHAR |TEXT
last_name.keyword |VARCHAR |KEYWORD
salary |INTEGER |INTEGER
; ;
showAlias showAlias

151
x-pack/qa/sql/src/main/resources/command.csv-spec
View File

@ -200,89 +200,98 @@ test_alias_emp |ALIAS
describeSimpleLike describeSimpleLike
DESCRIBE LIKE 'test_emp'; DESCRIBE LIKE 'test_emp';
column:s | type:s column | type | mapping
birth_date | TIMESTAMP --------------------+---------------+---------------
dep | STRUCT birth_date |TIMESTAMP |DATE
dep.dep_id | VARCHAR dep |STRUCT |NESTED
dep.dep_name | VARCHAR dep.dep_id |VARCHAR |KEYWORD
dep.dep_name.keyword | VARCHAR dep.dep_name |VARCHAR |TEXT
dep.from_date | TIMESTAMP dep.dep_name.keyword|VARCHAR |KEYWORD
dep.to_date | TIMESTAMP dep.from_date |TIMESTAMP |DATE
emp_no | INTEGER dep.to_date |TIMESTAMP |DATE
first_name | VARCHAR emp_no |INTEGER |INTEGER
first_name.keyword | VARCHAR extra_gender |VARCHAR |KEYWORD
gender | VARCHAR extra_no |INTEGER |INTEGER
hire_date | TIMESTAMP first_name |VARCHAR |TEXT
languages | TINYINT first_name.keyword |VARCHAR |KEYWORD
last_name | VARCHAR gender |VARCHAR |KEYWORD
last_name.keyword | VARCHAR hire_date |TIMESTAMP |DATE
salary | INTEGER languages |TINYINT |BYTE
last_name |VARCHAR |TEXT
last_name.keyword |VARCHAR |KEYWORD
salary |INTEGER |INTEGER
; ;
describeMultiLike describeMultiLike
DESCRIBE LIKE 'test_emp%'; DESCRIBE LIKE 'test_emp%';
column:s | type:s column | type | mapping
birth_date | TIMESTAMP --------------------+---------------+---------------
dep | STRUCT birth_date |TIMESTAMP |DATE
dep.dep_id | VARCHAR dep |STRUCT |NESTED
dep.dep_name | VARCHAR dep.dep_id |VARCHAR |KEYWORD
dep.dep_name.keyword | VARCHAR dep.dep_name |VARCHAR |TEXT
dep.from_date | TIMESTAMP dep.dep_name.keyword|VARCHAR |KEYWORD
dep.to_date | TIMESTAMP dep.from_date |TIMESTAMP |DATE
emp_no | INTEGER dep.to_date |TIMESTAMP |DATE
first_name | VARCHAR emp_no |INTEGER |INTEGER
first_name.keyword | VARCHAR extra_gender |VARCHAR |KEYWORD
gender | VARCHAR extra_no |INTEGER |INTEGER
hire_date | TIMESTAMP first_name |VARCHAR |TEXT
languages | TINYINT first_name.keyword |VARCHAR |KEYWORD
last_name | VARCHAR gender |VARCHAR |KEYWORD
last_name.keyword | VARCHAR hire_date |TIMESTAMP |DATE
salary | INTEGER languages |TINYINT |BYTE
last_name |VARCHAR |TEXT
last_name.keyword |VARCHAR |KEYWORD
salary |INTEGER |INTEGER
; ;
describeSimpleIdentifier describeSimpleIdentifier
DESCRIBE "test_emp"; DESCRIBE "test_emp";
column:s | type:s column | type | mapping
birth_date | TIMESTAMP --------------------+---------------+---------------
dep | STRUCT birth_date |TIMESTAMP |DATE
dep.dep_id | VARCHAR dep |STRUCT |NESTED
dep.dep_name | VARCHAR dep.dep_id |VARCHAR |KEYWORD
dep.dep_name.keyword | VARCHAR dep.dep_name |VARCHAR |TEXT
dep.from_date | TIMESTAMP dep.dep_name.keyword|VARCHAR |KEYWORD
dep.to_date | TIMESTAMP dep.from_date |TIMESTAMP |DATE
emp_no | INTEGER dep.to_date |TIMESTAMP |DATE
first_name | VARCHAR emp_no |INTEGER |INTEGER
first_name.keyword | VARCHAR first_name |VARCHAR |TEXT
gender | VARCHAR first_name.keyword |VARCHAR |KEYWORD
hire_date | TIMESTAMP gender |VARCHAR |KEYWORD
languages | TINYINT hire_date |TIMESTAMP |DATE
last_name | VARCHAR languages |TINYINT |BYTE
last_name.keyword | VARCHAR last_name |VARCHAR |TEXT
salary | INTEGER last_name.keyword |VARCHAR |KEYWORD
salary |INTEGER |INTEGER
; ;
describeIncludeExcludeIdentifier // NB: need to pursue how the resolution is done
DESCRIBE "test_emp*,-test_emp_*"; // should aliases be included or excluded?
describeIncludeExcludeIdentifier-Ignore
DESCRIBE "test_*,-test_alias*";
column:s | type:s column | type | mapping
birth_date | TIMESTAMP --------------------+---------------+---------------
dep | STRUCT birth_date |TIMESTAMP |DATE
dep.dep_id | VARCHAR dep |STRUCT |NESTED
dep.dep_name | VARCHAR dep.dep_id |VARCHAR |KEYWORD
dep.dep_name.keyword | VARCHAR dep.dep_name |VARCHAR |TEXT
dep.from_date | TIMESTAMP dep.dep_name.keyword|VARCHAR |KEYWORD
dep.to_date | TIMESTAMP dep.from_date |TIMESTAMP |DATE
emp_no | INTEGER dep.to_date |TIMESTAMP |DATE
first_name | VARCHAR emp_no |INTEGER |INTEGER
first_name.keyword | VARCHAR first_name |VARCHAR |TEXT
gender | VARCHAR first_name.keyword |VARCHAR |KEYWORD
hire_date | TIMESTAMP gender |VARCHAR |KEYWORD
languages | TINYINT hire_date |TIMESTAMP |DATE
last_name | VARCHAR languages |TINYINT |BYTE
last_name.keyword | VARCHAR last_name |VARCHAR |TEXT
salary | INTEGER last_name.keyword |VARCHAR |KEYWORD
salary |INTEGER |INTEGER
; ;

72
x-pack/qa/sql/src/main/resources/docs.csv-spec
View File

@ -12,24 +12,24 @@ describeTable
// tag::describeTable // tag::describeTable
DESCRIBE emp; DESCRIBE emp;
column | type column | type | mapping
--------------------+--------------- --------------------+---------------+---------------
birth_date |TIMESTAMP birth_date |TIMESTAMP |DATE
dep |STRUCT dep |STRUCT |NESTED
dep.dep_id |VARCHAR dep.dep_id |VARCHAR |KEYWORD
dep.dep_name |VARCHAR dep.dep_name |VARCHAR |TEXT
dep.dep_name.keyword|VARCHAR dep.dep_name.keyword|VARCHAR |KEYWORD
dep.from_date |TIMESTAMP dep.from_date |TIMESTAMP |DATE
dep.to_date |TIMESTAMP dep.to_date |TIMESTAMP |DATE
emp_no |INTEGER emp_no |INTEGER |INTEGER
first_name |VARCHAR first_name |VARCHAR |TEXT
first_name.keyword |VARCHAR first_name.keyword |VARCHAR |KEYWORD
gender |VARCHAR gender |VARCHAR |KEYWORD
hire_date |TIMESTAMP hire_date |TIMESTAMP |DATE
languages |TINYINT languages |TINYINT |BYTE
last_name |VARCHAR last_name |VARCHAR |TEXT
last_name.keyword |VARCHAR last_name.keyword |VARCHAR |KEYWORD
salary |INTEGER salary |INTEGER |INTEGER
// end::describeTable // end::describeTable
; ;
@ -51,24 +51,24 @@ showColumns
// tag::showColumns // tag::showColumns
SHOW COLUMNS IN emp; SHOW COLUMNS IN emp;
column | type column | type | mapping
--------------------+--------------- --------------------+---------------+---------------
birth_date |TIMESTAMP birth_date |TIMESTAMP |DATE
dep |STRUCT dep |STRUCT |NESTED
dep.dep_id |VARCHAR dep.dep_id |VARCHAR |KEYWORD
dep.dep_name |VARCHAR dep.dep_name |VARCHAR |TEXT
dep.dep_name.keyword|VARCHAR dep.dep_name.keyword|VARCHAR |KEYWORD
dep.from_date |TIMESTAMP dep.from_date |TIMESTAMP |DATE
dep.to_date |TIMESTAMP dep.to_date |TIMESTAMP |DATE
emp_no |INTEGER emp_no |INTEGER |INTEGER
first_name |VARCHAR first_name |VARCHAR |TEXT
first_name.keyword |VARCHAR first_name.keyword |VARCHAR |KEYWORD
gender |VARCHAR gender |VARCHAR |KEYWORD
hire_date |TIMESTAMP hire_date |TIMESTAMP |DATE
languages |TINYINT languages |TINYINT |BYTE
last_name |VARCHAR last_name |VARCHAR |TEXT
last_name.keyword |VARCHAR last_name.keyword |VARCHAR |KEYWORD
salary |INTEGER salary |INTEGER |INTEGER
// end::showColumns // end::showColumns
; ;

36
x-pack/qa/sql/src/main/resources/nested.csv-spec
View File

@ -6,24 +6,24 @@
describeParent describeParent
DESCRIBE test_emp; DESCRIBE test_emp;
column | type column | type | mapping
--------------------+---------------+---------------
birth_date | TIMESTAMP birth_date |TIMESTAMP |DATE
dep | STRUCT dep |STRUCT |NESTED
dep.dep_id | VARCHAR dep.dep_id |VARCHAR |KEYWORD
dep.dep_name | VARCHAR dep.dep_name |VARCHAR |TEXT
dep.dep_name.keyword | VARCHAR dep.dep_name.keyword|VARCHAR |KEYWORD
dep.from_date | TIMESTAMP dep.from_date |TIMESTAMP |DATE
dep.to_date | TIMESTAMP dep.to_date |TIMESTAMP |DATE
emp_no | INTEGER emp_no |INTEGER |INTEGER
first_name | VARCHAR first_name |VARCHAR |TEXT
first_name.keyword | VARCHAR first_name.keyword |VARCHAR |KEYWORD
gender | VARCHAR gender |VARCHAR |KEYWORD
hire_date | TIMESTAMP hire_date |TIMESTAMP |DATE
languages | TINYINT languages |TINYINT |BYTE
last_name | VARCHAR last_name |VARCHAR |TEXT
last_name.keyword | VARCHAR last_name.keyword |VARCHAR |KEYWORD
salary | INTEGER salary |INTEGER |INTEGER
; ;
// disable until we figure out how to use field names with . in their name // disable until we figure out how to use field names with . in their name