Adds docs for the REST API, translate API, the CLI, and JDBC.

Next we need to add more example queries and documentation for our
extensions.

Original commit: elastic/x-pack-elasticsearch@ed6d1360d2
This commit is contained in:
Nik Everett 2017-10-30 17:23:27 +00:00 committed by GitHub
parent 66719f7a92
commit d933b1b48b
14 changed files with 395 additions and 28 deletions

View File

@ -200,3 +200,72 @@ setups['my_inactive_watch'] = '''
'''
setups['my_active_watch'] = setups['my_inactive_watch'].replace(
'active: false', 'active: true')
setups['library'] = '''
- do:
indices.create:
index: library
body:
settings:
number_of_shards: 1
number_of_replicas: 1
mappings:
book:
properties:
name:
type: keyword
author:
type: keyword
release_date:
type: date
page_count:
type: short
- do:
bulk:
index: library
type: book
refresh: true
body: |
{"index":{"_id": "Leviathan Wakes"}}
{"name": "Leviathan Wakes", "author": "James S.A. Corey", "release_date": "2011-06-02", "page_count": 561}
{"index":{"_id": "Hyperion"}}
{"name": "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482}
{"index":{"_id": "Dune"}}
{"name": "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604}
{"index":{"_id": "Consider Phlebas"}}
{"name": "Consider Phlebas", "author": "Iain M. Banks", "release_date": "1987-04-23", "page_count": 471}
{"index":{"_id": "Pandora's Star"}}
{"name": "Pandora's Star", "author": "Peter F. Hamilton", "release_date": "2004-03-02", "page_count": 768}
{"index":{"_id": "Revelation Space"}}
{"name": "Revelation Space", "author": "Alastair Reynolds", "release_date": "2000-03-15", "page_count": 585}
{"index":{"_id": "A Fire Upon the Deep"}}
{"name": "A Fire Upon the Deep", "author": "Vernor Vinge", "release_date": "1992-06-01", "page_count": 613}
{"index":{"_id": "Ender's Game"}}
{"name": "Ender's Game", "author": "Orson Scott Card", "release_date": "1985-06-01", "page_count": 324}
{"index":{"_id": "1984"}}
{"name": "1984", "author": "George Orwell", "release_date": "1985-06-01", "page_count": 328}
{"index":{"_id": "Fahrenheit 451"}}
{"name": "Fahrenheit 451", "author": "Ray Bradbury", "release_date": "1953-10-15", "page_count": 227}
{"index":{"_id": "Brave New World"}}
{"name": "Brave New World", "author": "Aldous Huxley", "release_date": "1932-06-01", "page_count": 268}
{"index":{"_id": "Foundation"}}
{"name": "Foundation", "author": "Isaac Asimov", "release_date": "1951-06-01", "page_count": 224}
{"index":{"_id": "The Giver"}}
{"name": "The Giver", "author": "Lois Lowry", "release_date": "1993-04-26", "page_count": 208}
{"index":{"_id": "Slaughterhouse-Five"}}
{"name": "Slaughterhouse-Five", "author": "Kurt Vonnegut", "release_date": "1969-06-01", "page_count": 275}
{"index":{"_id": "The Hitchhiker's Guide to the Galaxy"}}
{"name": "The Hitchhiker's Guide to the Galaxy", "author": "", "release_date": "1979-10-12", "page_count": 180}
{"index":{"_id": "Snow Crash"}}
{"name": "Snow Crash", "author": "Neal Stephenson", "release_date": "1992-06-01", "page_count": 470}
{"index":{"_id": "Neuromancer"}}
{"name": "Neuromancer", "author": "William Gibson", "release_date": "1984-07-01", "page_count": 271}
{"index":{"_id": "The Handmaid's Tale"}}
{"name": "The Handmaid's Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311}
{"index":{"_id": "Starship Troopers"}}
{"name": "Starship Troopers", "author": "Robert A. Heinlein", "release_date": "1959-12-01", "page_count": 335}
{"index":{"_id": "The Left Hand of Darkness"}}
{"name": "The Left Hand of Darkness", "author": "Ursula K. Le Guin", "release_date": "1969-06-01", "page_count": 304}
{"index":{"_id": "The Moon is a Harsh Mistress"}}
{"name": "The Moon is a Harsh Mistress", "author": "Robert A. Heinlein", "release_date": "1966-04-01", "page_count": 288}
'''

View File

@ -2,17 +2,33 @@
[[xpack-sql]]
= SQL Access
:jdbc-tests: {docdir}/../../sql/jdbc/src/test/resources
:sql-tests: {docdir}/../../qa/sql
:sql-specs: {sql-tests}/src/main/resources
:jdbc-tests: {sql-tests}/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc
:security-tests: {sql-tests}/security/src/test/java/org/elasticsearch/xpack/qa/sql/security
[partintro]
--
Overview
X-Pack includes a SQL feature to execute SQL against Elasticsearch
indices and return tabular results. There are four main components:
<<sql-rest,REST API>>::
Accepts SQL in a JSON document, executes it, and returns the
results.
<<sql-translate,Translate API>>::
Accepts SQL in a JSON document and translates it into a native
Elasticsearch query and returns that.
<<sql-cli,CLI>>::
Command line application that connects to Elasticsearch to excute
SQL and print tabular results.
<<sql-jdbc,JDBC>>::
A JDBC driver for Elasticsearch.
--
include::sql-query-dsl.asciidoc[]
include::sql-rest.asciidoc[]
include::sql-jdbc.asciidoc[]
include::sql-translate.asciidoc[]
include::sql-cli.asciidoc[]
include::sql-jdbc.asciidoc[]
include::sql-query-dsl.asciidoc[]
:jdbc-tests!:

View File

@ -2,4 +2,38 @@
[[sql-cli]]
== SQL CLI
Content
The SQL CLI is a stand alone Java application for quick interaction
with X-Pack SQL. You can run it like this:
["source","bash",subs="attributes,callouts"]
--------------------------------------------------
$ java -jar cli-{version}.jar
--------------------------------------------------
You can pass the URL of the Elasticsearch instance to connect to as
the first parameter:
["source","bash",subs="attributes,callouts"]
--------------------------------------------------
$ java -jar cli-{version}.jar https://some.server:9200
--------------------------------------------------
The cli jar is entirely stand alone and can be moved whereever it is
needed.
Once the CLI is running you can use any <<sql-query-dsl,query>> that
Elasticsearch supports:
[source,sqlcli]
--------------------------------------------------
sql> SELECT * FROM library WHERE page_count > 500 ORDER BY page_count DESC;
author | name | page_count
----------------------------+-----------------------+---------------
Victor Hugo |Les Misérables |1463
Miguel De Cervantes Saavedra|Don Quixote |1072
Miguel De Cervantes Saavedra|Don Quixote |1072
Herman Melville |Moby-Dick or, The Whale|720
Charles Dickens |Oliver Twist |608
--------------------------------------------------
// TODO it'd be lovely to be able to assert that this is correct but
// that is probably more work then it is worth right now.

View File

@ -2,4 +2,30 @@
[[sql-jdbc]]
== SQL JDBC
Content
Elasticsearch's SQL jdbc driver is a fully featured JDBC driver
for Elasticsearch. You can connect to it with:
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{jdbc-tests}/JdbcIntegrationTestCase.java[connect]
--------------------------------------------------
<1> The server and port on which Elasticsearch is listening for
HTTP traffic. The port is usually 9200.
<2> Properties for connecting to Elasticsearch. An empty `Properties`
instance is fine for unsecured Elasticsearch.
To connect to a secured Elasticsearch server the `Properties`
should look like:
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{security-tests}/JdbcSecurityIT.java[admin_properties]
--------------------------------------------------
Once you have the connection you can use it like any other JDBC
connection. For example:
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{jdbc-tests}/SimpleExampleTestCase.java[simple_example]
--------------------------------------------------

View File

@ -9,5 +9,5 @@ Example!
["source","sql",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{jdbc-tests}/select.sql-spec[wildcardWithOrder]
include-tagged::{sql-spec}/select.sql-spec[wildcardWithOrder]
--------------------------------------------------

View File

@ -2,4 +2,89 @@
[[sql-rest]]
== SQL REST API
Content
The SQL REST API accepts SQL in a JSON document, executes it,
and returns the results. For example:
[source,js]
--------------------------------------------------
POST /_sql
{
"query": "SELECT * FROM library ORDER BY page_count DESC",
"fetch_size": 5
}
--------------------------------------------------
// CONSOLE
// TEST[setup:library]
Which returns:
[source,js]
--------------------------------------------------
{
"columns": [
{"name": "author", "type": "keyword"},
{"name": "name", "type": "keyword"},
{"name": "page_count", "type": "short"},
{"name": "release_date", "type": "date"}
],
"size": 5,
"rows": [
["Peter F. Hamilton", "Pandora's Star", 768, 1078185600000],
["Vernor Vinge", "A Fire Upon the Deep", 613, 707356800000],
["Frank Herbert", "Dune", 604, -144720000000],
["Alastair Reynolds", "Revelation Space", 585, 953078400000],
["James S.A. Corey", "Leviathan Wakes", 561, 1306972800000]
],
"cursor": "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl+v///w8="
}
--------------------------------------------------
// TESTRESPONSE[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl\+v\/\/\/w8=/$body.cursor/]
You can continue to the next page by sending back the `cursor` field:
[source,js]
--------------------------------------------------
POST /_sql
{
"cursor": "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWYUpOYklQMHhRUEtld3RsNnFtYU1hQQ==:BAFmBGRhdGUBZgVsaWtlcwFzB21lc3NhZ2UBZgR1c2Vy9f///w8="
}
--------------------------------------------------
// CONSOLE
// TEST[continued]
// TEST[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWYUpOYklQMHhRUEtld3RsNnFtYU1hQQ==:BAFmBGRhdGUBZgVsaWtlcwFzB21lc3NhZ2UBZgR1c2Vy9f\/\/\/w8=/$body.cursor/]
Which looks like:
[source,js]
--------------------------------------------------
{
"size" : 5,
"rows" : [
["Dan Simmons", "Hyperion", 482, 612144000000],
["Iain M. Banks", "Consider Phlebas", 471, 546134400000],
["Neal Stephenson", "Snow Crash", 470, 707356800000],
["Robert A. Heinlein", "Starship Troopers", 335, -318297600000],
["George Orwell", "1984", 328, 486432000000]
],
"cursor" : "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWODRMaXBUaVlRN21iTlRyWHZWYUdrdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl9f///w8="
}
--------------------------------------------------
// TESTRESPONSE[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWODRMaXBUaVlRN21iTlRyWHZWYUdrdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl9f\/\/\/w8=/$body.cursor/]
Note that the `column` object is only part of the first page.
You've reached the last page when there is no `cursor` returned
in the results. Like Elasticsearch's <<search-request-scroll,scroll>>,
SQL may keep state in Elasticsearch to support the cursor. Unlike
scroll, receiving the last page is enough to guarantee that the
Elasticsearch state is cleared. For now, that is the only way to
clear the state.
[[sql-rest-fields]]
In addition to the `query` and `cursor` fields, the request can
contain `fetch_size` and `time_zone`. `fetch_size` is a hint for how
many results to return in each page. SQL might chose to return more
or fewer results though. `time_zone` is the time zone to use for date
functions and date parsing. `time_zone` defaults to `utc` and can take
any values documented
http://www.joda.org/joda-time/apidocs/org/joda/time/DateTimeZone.html[here].

View File

@ -0,0 +1,48 @@
[role="xpack"]
[[sql-translate]]
== SQL Translate API
The SQL Translate API accepts SQL in a JSON document and translates it
into native Elasticsearch queries. For example:
[source,js]
--------------------------------------------------
POST /_sql/translate
{
"query": "SELECT * FROM library ORDER BY page_count DESC",
"fetch_size": 10
}
--------------------------------------------------
// CONSOLE
// TEST[setup:library]
Which returns:
[source,js]
--------------------------------------------------
{
"size" : 10,
"docvalue_fields" : [
"author",
"name",
"page_count",
"release_date"
],
"sort" : [
{
"page_count" : {
"order" : "desc"
}
}
]
}
--------------------------------------------------
// TESTRESPONSE
Which is the request that SQL will run to provide the results.
In this case, SQL will use the <<search-request-scroll,scroll>>
API. If the result contained an aggregation then SQL would use
the normal <<search-request-body,search>> API.
The request body accepts all of the <<sql-rest-fields,fields>> that
the <<sql-rest,SQL REST API>> accepts except `cursor`.

View File

@ -0,0 +1,11 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.qa.sql.multinode;
import org.elasticsearch.xpack.qa.sql.jdbc.SimpleExampleTestCase;
public class JdbcSimpleExampleIT extends SimpleExampleTestCase {
}

View File

@ -0,0 +1,11 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.qa.sql.nosecurity;
import org.elasticsearch.xpack.qa.sql.jdbc.SimpleExampleTestCase;
public class JdbcSimpleExampleIT extends SimpleExampleTestCase {
}

View File

@ -21,17 +21,23 @@ import static org.hamcrest.Matchers.containsString;
public class JdbcSecurityIT extends SqlSecurityTestCase {
static Properties adminProperties() {
Properties prop = new Properties();
prop.put("user", "test_admin");
prop.put("pass", "x-pack-test-password");
return prop;
// tag::admin_properties
Properties properties = new Properties();
properties.put("user", "test_admin");
properties.put("pass", "x-pack-test-password");
// end::admin_properties
return properties;
}
static Connection es(Properties properties) throws SQLException {
return DriverManager.getConnection("jdbc:es://" + elasticsearchAddress(), properties);
}
private static class JdbcActions implements Actions {
@Override
public void queryWorksAsAdmin() throws Exception {
try (Connection h2 = LocalH2.anonymousDb();
Connection es = DriverManager.getConnection(elasticsearchAddress(), adminProperties())) {
Connection es = es(adminProperties())) {
h2.createStatement().executeUpdate("CREATE TABLE test (a BIGINT, b BIGINT, c BIGINT)");
h2.createStatement().executeUpdate("INSERT INTO test (a, b, c) VALUES (1, 2, 3), (4, 5, 6)");
@ -42,8 +48,8 @@ public class JdbcSecurityIT extends SqlSecurityTestCase {
@Override
public void expectMatchesAdmin(String adminSql, String user, String userSql) throws Exception {
try (Connection admin = DriverManager.getConnection(elasticsearchAddress(), adminProperties());
Connection other = DriverManager.getConnection(elasticsearchAddress(), userProperties(user))) {
try (Connection admin = es(adminProperties());
Connection other = es(userProperties(user))) {
ResultSet expected = admin.createStatement().executeQuery(adminSql);
assertResultSets(expected, other.createStatement().executeQuery(userSql));
}
@ -51,8 +57,8 @@ public class JdbcSecurityIT extends SqlSecurityTestCase {
@Override
public void expectScrollMatchesAdmin(String adminSql, String user, String userSql) throws Exception {
try (Connection admin = DriverManager.getConnection(elasticsearchAddress(), adminProperties());
Connection other = DriverManager.getConnection(elasticsearchAddress(), userProperties(user))) {
try (Connection admin = es(adminProperties());
Connection other = es(userProperties(user))) {
Statement adminStatement = admin.createStatement();
adminStatement.setFetchSize(1);
Statement otherStatement = other.createStatement();
@ -64,7 +70,7 @@ public class JdbcSecurityIT extends SqlSecurityTestCase {
@Override
public void expectDescribe(Map<String, String> columns, String user) throws Exception {
try (Connection h2 = LocalH2.anonymousDb();
Connection es = DriverManager.getConnection(elasticsearchAddress(), userProperties(user))) {
Connection es = es(userProperties(user))) {
// h2 doesn't have the same sort of DESCRIBE that we have so we emulate it
h2.createStatement().executeUpdate("CREATE TABLE mock (column VARCHAR, type VARCHAR)");
StringBuilder insert = new StringBuilder();
@ -88,7 +94,7 @@ public class JdbcSecurityIT extends SqlSecurityTestCase {
@Override
public void expectShowTables(List<String> tables, String user) throws Exception {
try (Connection h2 = LocalH2.anonymousDb();
Connection es = DriverManager.getConnection(elasticsearchAddress(), userProperties(user))) {
Connection es = es(userProperties(user))) {
// h2 doesn't spit out the same columns we do so we emulate
h2.createStatement().executeUpdate("CREATE TABLE mock (table VARCHAR)");
StringBuilder insert = new StringBuilder();
@ -112,7 +118,7 @@ public class JdbcSecurityIT extends SqlSecurityTestCase {
@Override
public void expectForbidden(String user, String sql) throws Exception {
SQLException e;
try (Connection connection = DriverManager.getConnection(elasticsearchAddress(), userProperties(user))) {
try (Connection connection = es(userProperties(user))) {
e = expectThrows(SQLException.class, () -> connection.createStatement().executeQuery(sql));
}
assertThat(e.getMessage(), containsString("is unauthorized for user [" + user + "]"));
@ -121,7 +127,7 @@ public class JdbcSecurityIT extends SqlSecurityTestCase {
@Override
public void expectUnknownColumn(String user, String sql, String column) throws Exception {
SQLException e;
try (Connection connection = DriverManager.getConnection(elasticsearchAddress(), userProperties(user))) {
try (Connection connection = es(userProperties(user))) {
e = expectThrows(SQLException.class, () -> connection.createStatement().executeQuery(sql));
}
assertThat(e.getMessage(), containsString("Unknown column [" + column + "]"));

View File

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.qa.sql.security;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.xpack.qa.sql.jdbc.SimpleExampleTestCase;
import java.util.Properties;
public class JdbcSimpleExampleIT extends SimpleExampleTestCase {
@Override
protected Settings restClientSettings() {
return RestSqlIT.securitySettings();
}
@Override
protected Properties connectionProperties() {
return JdbcSecurityIT.adminProperties();
}
}

View File

@ -48,14 +48,20 @@ public abstract class JdbcIntegrationTestCase extends ESRestTestCase {
public static String elasticsearchAddress() {
String cluster = System.getProperty("tests.rest.cluster");
// JDBC only supports a single node at a time so we just give it one.
return "jdbc:es://" + cluster.split(",")[0];
return cluster.split(",")[0];
/* This doesn't include "jdbc:es://" because we want the example in
* esJdbc to be obvious. */
}
public Connection esJdbc() throws SQLException {
if (EMBED_SQL) {
return EMBEDDED_SERVER.connection();
}
return DriverManager.getConnection(elasticsearchAddress(), connectionProperties());
// tag::connect
String address = "jdbc:es://" + elasticsearchAddress(); // <1>
Properties connectionProperties = connectionProperties(); // <2>
return DriverManager.getConnection(address, connectionProperties);
// end::connect
}
public static void index(String index, CheckedConsumer<XContentBuilder, IOException> body) throws IOException {
@ -81,4 +87,4 @@ public abstract class JdbcIntegrationTestCase extends ESRestTestCase {
protected Properties connectionProperties() {
return new Properties();
}
}
}

View File

@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.qa.sql.jdbc;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
public class SimpleExampleTestCase extends JdbcIntegrationTestCase {
public void testSimpleExample() throws Exception {
index("library", builder -> {
builder.field("name", "Don Quixote");
builder.field("page_count", 1072);
});
try (Connection connection = esJdbc()) {
// tag::simple_example
try (Statement statement = connection.createStatement();
ResultSet results = statement.executeQuery(
"SELECT name, page_count FROM library ORDER BY page_count DESC LIMIT 1")) {
assertTrue(results.next());
assertEquals("Don Quixote", results.getString(1));
assertEquals(1072, results.getInt(2));
assertFalse(results.next());
}
// end::simple_example
}
}
}

View File

@ -36,7 +36,8 @@ public class SqlResponse extends ActionResponse implements ToXContentObject {
public SqlResponse(Cursor cursor, long size, int columnCount, @Nullable List<ColumnInfo> columns, List<List<Object>> rows) {
this.cursor = cursor;
this.size = size;
this.size = size; // NOCOMMIT Probably should be removed.
// Size isn't the total number of results like ES uses, it is the size of the rows list.
this.columnCount = columnCount;
this.columns = columns;
this.rows = rows;
@ -218,14 +219,14 @@ public class SqlResponse extends ActionResponse implements ToXContentObject {
}
/**
* The type of the column as it would be returned by a JDBC driver.
* The type of the column as it would be returned by a JDBC driver.
*/
public JDBCType jdbcType() {
return jdbcType;
}
/**
* Used by JDBC
* Used by JDBC
*/
public int displaySize() {
return displaySize;
@ -253,4 +254,4 @@ public class SqlResponse extends ActionResponse implements ToXContentObject {
return Strings.toString(this);
}
}
}
}