Merge branch 'master' into index-lifecycle

libs/nio/src/main/java/org/elasticsearch/nio/ChannelContext.java

@@ -66,7 +66,7 @@ public abstract class ChannelContext<S extends SelectableChannel & NetworkChanne
      * @throws IOException during channel / context close
      */
     public void closeFromSelector() throws IOException {
-        if (closeContext.isDone() == false) {
+        if (isOpen()) {
             try {
                 rawChannel.close();
                 closeContext.complete(null);

libs/nio/src/main/java/org/elasticsearch/nio/EventHandler.java

@@ -159,8 +159,7 @@ public class EventHandler {
     }
 
     /**
-     * This method is called after ready events (READ, ACCEPT, WRITE, CONNECT) have been handled for a
-     * channel.
+     * This method is called after events (READ, WRITE, CONNECT) have been handled for a channel.
      *
      * @param context that was handled
      */

libs/nio/src/main/java/org/elasticsearch/nio/NioSelector.java

@@ -43,9 +43,6 @@ import java.util.stream.Collectors;
  * {@link #runLoop()}, the selector will run until {@link #close()} is called. This instance handles closing
  * of channels. Users should call {@link #queueChannelClose(NioChannel)} to schedule a channel for close by
  * this selector.
- * <p>
- * Children of this class should implement the specific {@link #processKey(SelectionKey)},
- * {@link #preSelect()}, and {@link #cleanup()} functionality.
  */
 public class NioSelector implements Closeable {
 
@@ -65,7 +62,7 @@ public class NioSelector implements Closeable {
         this(eventHandler, Selector.open());
     }
 
-    public NioSelector(EventHandler eventHandler, Selector selector) throws IOException {
+    public NioSelector(EventHandler eventHandler, Selector selector) {
         this.selector = selector;
         this.eventHandler = eventHandler;
     }
@@ -165,7 +162,7 @@ public class NioSelector implements Closeable {
     }
 
     void cleanupAndCloseChannels() {
-        cleanup();
+        cleanupPendingWrites();
         channelsToClose.addAll(channelsToRegister);
         channelsToRegister.clear();
         channelsToClose.addAll(selector.keys().stream().map(sk -> (ChannelContext<?>) sk.attachment()).collect(Collectors.toList()));
@@ -234,16 +231,6 @@ public class NioSelector implements Closeable {
         handleQueuedWrites();
     }
 
-    /**
-     * Called once as the selector is being closed.
-     */
-    void cleanup() {
-        WriteOperation op;
-        while ((op = queuedWrites.poll()) != null) {
-            executeFailedListener(op.getListener(), new ClosedSelectorException());
-        }
-    }
-
     /**
      * Queues a write operation to be handled by the event loop. This can be called by any thread and is the
      * api available for non-selector threads to schedule writes.
@@ -284,20 +271,31 @@ public class NioSelector implements Closeable {
     }
 
     /**
-     * Queues a write operation directly in a channel's buffer. Channel buffers are only safe to be accessed
-     * by the selector thread. As a result, this method should only be called by the selector thread.
+     * Queues a write operation directly in a channel's buffer. If this channel does not have pending writes
+     * already, the channel will be flushed. Channel buffers are only safe to be accessed by the selector
+     * thread. As a result, this method should only be called by the selector thread. If this channel does
+     * not have pending writes already, the channel will be flushed.
      *
      * @param writeOperation to be queued in a channel's buffer
      */
-    public void queueWriteInChannelBuffer(WriteOperation writeOperation) {
+    public void writeToChannel(WriteOperation writeOperation) {
         assertOnSelectorThread();
         SocketChannelContext context = writeOperation.getChannel();
+        // If the channel does not currently have anything that is ready to flush, we should flush after
+        // the write operation is queued.
+        boolean shouldFlushAfterQueuing = context.readyForFlush() == false;
         try {
             SelectionKeyUtils.setWriteInterested(context.getSelectionKey());
             context.queueWriteOperation(writeOperation);
         } catch (Exception e) {
+            shouldFlushAfterQueuing = false;
             executeFailedListener(writeOperation.getListener(), e);
         }
+
+        if (shouldFlushAfterQueuing) {
+            handleWrite(context);
+            eventHandler.postHandling(context);
+        }
     }
 
     /**
@@ -332,6 +330,13 @@ public class NioSelector implements Closeable {
         }
     }
 
+    private void cleanupPendingWrites() {
+        WriteOperation op;
+        while ((op = queuedWrites.poll()) != null) {
+            executeFailedListener(op.getListener(), new ClosedSelectorException());
+        }
+    }
+
     private void wakeup() {
         // TODO: Do we need the wakeup optimizations that some other libraries use?
         selector.wakeup();
@@ -394,7 +399,7 @@ public class NioSelector implements Closeable {
         WriteOperation writeOperation;
         while ((writeOperation = queuedWrites.poll()) != null) {
             if (writeOperation.getChannel().isOpen()) {
-                queueWriteInChannelBuffer(writeOperation);
+                writeToChannel(writeOperation);
             } else {
                 executeFailedListener(writeOperation.getListener(), new ClosedChannelException());
             }

libs/nio/src/main/java/org/elasticsearch/nio/SocketChannelContext.java

@@ -135,7 +135,7 @@ public abstract class SocketChannelContext extends ChannelContext<SocketChannel>
             return;
         }
 
-        selector.queueWriteInChannelBuffer(writeOperation);
+        selector.writeToChannel(writeOperation);
     }
 
     public void queueWriteOperation(WriteOperation writeOperation) {
@@ -164,7 +164,7 @@ public abstract class SocketChannelContext extends ChannelContext<SocketChannel>
     @Override
     public void closeFromSelector() throws IOException {
         getSelector().assertOnSelectorThread();
-        if (channel.isOpen()) {
+        if (isOpen()) {
             ArrayList<IOException> closingExceptions = new ArrayList<>(3);
             try {
                 super.closeFromSelector();

libs/nio/src/test/java/org/elasticsearch/nio/NioSelectorTests.java

@@ -262,11 +262,28 @@ public class NioSelectorTests extends ESTestCase {
     public void testQueueDirectlyInChannelBufferSuccessful() throws Exception {
         WriteOperation writeOperation = new FlushReadyWrite(channelContext, buffers, listener);
 
-        assertTrue((selectionKey.interestOps() & SelectionKey.OP_WRITE) == 0);
+        assertEquals(0, (selectionKey.interestOps() & SelectionKey.OP_WRITE));
 
-        selector.queueWriteInChannelBuffer(writeOperation);
+        when(channelContext.readyForFlush()).thenReturn(true);
+        selector.writeToChannel(writeOperation);
 
         verify(channelContext).queueWriteOperation(writeOperation);
+        verify(eventHandler, times(0)).handleWrite(channelContext);
+        verify(eventHandler, times(0)).postHandling(channelContext);
+        assertTrue((selectionKey.interestOps() & SelectionKey.OP_WRITE) != 0);
+    }
+
+    public void testShouldFlushIfNoPendingFlushes() throws Exception {
+        WriteOperation writeOperation = new FlushReadyWrite(channelContext, buffers, listener);
+
+        assertEquals(0, (selectionKey.interestOps() & SelectionKey.OP_WRITE));
+
+        when(channelContext.readyForFlush()).thenReturn(false);
+        selector.writeToChannel(writeOperation);
+
+        verify(channelContext).queueWriteOperation(writeOperation);
+        verify(eventHandler).handleWrite(channelContext);
+        verify(eventHandler).postHandling(channelContext);
         assertTrue((selectionKey.interestOps() & SelectionKey.OP_WRITE) != 0);
     }
 
@@ -277,10 +294,13 @@ public class NioSelectorTests extends ESTestCase {
         CancelledKeyException cancelledKeyException = new CancelledKeyException();
 
         when(channelContext.getSelectionKey()).thenReturn(selectionKey);
+        when(channelContext.readyForFlush()).thenReturn(false);
         when(selectionKey.interestOps(anyInt())).thenThrow(cancelledKeyException);
-        selector.queueWriteInChannelBuffer(writeOperation);
+        selector.writeToChannel(writeOperation);
 
         verify(channelContext, times(0)).queueWriteOperation(writeOperation);
+        verify(eventHandler, times(0)).handleWrite(channelContext);
+        verify(eventHandler, times(0)).postHandling(channelContext);
         verify(listener).accept(null, cancelledKeyException);
     }
 

libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java

@@ -170,7 +170,7 @@ public class SocketChannelContextTests extends ESTestCase {
         when(readWriteHandler.createWriteOperation(context, buffers, listener)).thenReturn(writeOperation);
         context.sendMessage(buffers, listener);
 
-        verify(selector).queueWriteInChannelBuffer(writeOpCaptor.capture());
+        verify(selector).writeToChannel(writeOpCaptor.capture());
         WriteOperation writeOp = writeOpCaptor.getValue();
 
         assertSame(writeOperation, writeOp);

x-pack/docs/en/sql/functions/index.asciidoc

@@ -1,9 +1,11 @@
 [[sql-functions]]
 == Functions and Operators
 
+{es-sql} provides a number of built-in operators and functions.
+
 === Comparison Operators
 
-Elasticsearch SQL supports the following comparison operators:
+{es-sql} supports the following comparison operators:
 
 * Equality (`=`)
 
@@ -12,7 +14,7 @@ Elasticsearch SQL supports the following comparison operators:
 include-tagged::{sql-specs}/filter.sql-spec[whereFieldEquality]
 --------------------------------------------------
 
-* Inequality (`<>` or `!=`)
+* Inequality (`<>` or `!=` or `<=>`)
 
 ["source","sql",subs="attributes,callouts,macros"]
 --------------------------------------------------
@@ -43,7 +45,7 @@ include-tagged::{sql-specs}/filter.sql-spec[whereIsNotNullAndIsNull]
 
 === Logical Operators
 
-Elasticsearch SQL supports the following logical operators:
+{es-sql} supports the following logical operators:
 
 * `AND`
 
@@ -69,7 +71,7 @@ include-tagged::{sql-specs}/filter.sql-spec[whereFieldEqualityNot]
 
 === Math Operators
 
-Elasticsearch SQL supports the following math operators:
+{es-sql} supports the following math operators:
 
 * Add (`+`)
 
@@ -106,7 +108,7 @@ include-tagged::{sql-specs}/arithmetic.sql-spec[multiply]
 include-tagged::{sql-specs}/arithmetic.sql-spec[divide]
 --------------------------------------------------
 
-* https://en.wikipedia.org/wiki/Modulo_operation[Modulo] (`%`)
+* https://en.wikipedia.org/wiki/Modulo_operation[Modulo] or Reminder(`%`)
 
 ["source","sql",subs="attributes,callouts,macros"]
 --------------------------------------------------
@@ -115,24 +117,48 @@ include-tagged::{sql-specs}/arithmetic.sql-spec[mod]
 
 
 === Math Functions
-==== Basic
 
-* https://en.wikipedia.org/wiki/Absolute_value[Absolute value] (`ABS`)
+All math and trigonometric functions require their input (where applicable)
+to be numeric.
+
+==== Generic
+
+* `ABS`
+
+https://en.wikipedia.org/wiki/Absolute_value[Absolute value], returns \[same type as input]
 
 ["source","sql",subs="attributes,callouts,macros"]
 --------------------------------------------------
 include-tagged::{sql-specs}/math.sql-spec[abs]
 --------------------------------------------------
 
+* `CBRT`
+
+https://en.wikipedia.org/wiki/Cube_root[Cube root], returns `double`
+
+// TODO make the example in the tests presentable
+
+* `CEIL`
+
+https://en.wikipedia.org/wiki/Floor_and_ceiling_functions[Ceiling], returns `double`
+
+* `CEILING`
+
+Same as `CEIL`
+
+// TODO make the example in the tests presentable
+
+* `E`
+
+https://en.wikipedia.org/wiki/E_%28mathematical_constant%29[Euler's number], returns `2.7182818284590452354`
+
+
 * https://en.wikipedia.org/wiki/Rounding#Round_half_up[Round] (`ROUND`)
 
 // TODO make the example in the tests presentable
 
 NOTE: This rounds "half up" meaning that `ROUND(-1.5)` results in `-1`.
 
-* https://en.wikipedia.org/wiki/Floor_and_ceiling_functions[Ceiling] (`CEIL`)
-
-// TODO make the example in the tests presentable
 
 * https://en.wikipedia.org/wiki/Floor_and_ceiling_functions[Floor] (`FLOOR`)
 
@@ -159,10 +185,6 @@ include-tagged::{sql-specs}/math.sql-spec[log10]
 include-tagged::{sql-specs}/math.sql-spec[sqrt]
 --------------------------------------------------
 
-* https://en.wikipedia.org/wiki/Cube_root[Cube root] (`CBRT`)
-
-// TODO make the example in the tests presentable
-
 * https://en.wikipedia.org/wiki/Exponential_function[e^x^] (`EXP`)
 
 ["source","sql",subs="attributes,callouts,macros"]

x-pack/docs/en/sql/getting-started.asciidoc

@@ -1,7 +1,7 @@
 [[sql-getting-started]]
 == Getting Started with SQL
 
-To start using Elasticsearch SQL, create
+To start using {es-sql}, create
 an index with some data to experiment with:
 
 [source,js]

x-pack/docs/en/sql/index.asciidoc

@@ -6,6 +6,7 @@
 :sql-specs: {sql-tests}/src/main/resources
 :jdbc-tests: {sql-tests}/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc
 :security-tests: {sql-tests}/security/src/test/java/org/elasticsearch/xpack/qa/sql/security
+:es-sql: Elasticsearch SQL
 
 [partintro]
 --
@@ -13,8 +14,12 @@
 experimental[]
 
 X-Pack includes a SQL feature to execute SQL against Elasticsearch
-indices and return tabular results. There are four main components:
+indices and return results in tabular format.
 
+<<sql-overview, Overview>>::
+    Overview of {es-sql} and its features.
+<<sql-getting-started, Getting Started>>::
+    Start using SQL right away in {es}
 <<sql-rest,REST API>>::
     Accepts SQL in a JSON document, executes it, and returns the
     results.
@@ -22,12 +27,19 @@ indices and return tabular results. There are four main components:
     Accepts SQL in a JSON document and translates it into a native
     Elasticsearch query and returns that.
 <<sql-cli,CLI>>::
-    Command line application that connects to Elasticsearch to execute
+    Command-line application that connects to {es} to execute
     SQL and print tabular results.
 <<sql-jdbc,JDBC>>::
-    A JDBC driver for Elasticsearch.
+    A JDBC driver for {es}.
+<<sql-functions,Functions and Operators>>::
+    List of functions and operators supported.
+<<sql-spec,SQL Language>>::
+    Overview of the {es-sql} language, such as data types, syntax and 
+    reserved keywords.
+
 --
 
+include::overview.asciidoc[]
 include::getting-started.asciidoc[]
 include::endpoints/index.asciidoc[]
 include::functions/index.asciidoc[]

x-pack/docs/en/sql/language/data-types.asciidoc

@@ -1,6 +1,87 @@
 [[sql-data-types]]
-=== Data Type and Mapping
+=== Data Types
+
+Most of {es} <<mapping-types, data types>> are available in {es-sql}, as indicated below:
+
+[cols="^,^,^",options="header"]
+
+|===
+| {es} type | SQL type | SQL precision
+
+3+h| Core types
+
+| <<null-value, `null`>>      | `null`      | 0
+| <<boolean, `boolean`>>      | `boolean`   | 1
+| <<number, `byte`>>          | `tinyint`   | 3
+| <<number, `short`>>         | `smallint`  | 5
+| <<number, `integer`>>       | `integer`   | 10
+| <<number, `long`>>          | `long`      | 19
+| <<number, `double`>>        | `double`    | 15
+| <<number, `float`>>         | `real`      | 7
+| <<number, `half_float`>>    | `float`     | 16
+| <<number, `scaled_float`>>  | `float`     | 19
+| <<keyword, `keyword`>>      | `varchar`   | based on <<ignore-above>>
+| <<text, `text`>>            | `varchar`   | 2,147,483,647
+| <<binary, `binary`>>        | `varbinary` | 2,147,483,647
+| <<date, `date`>>            | `timestamp` | 24
+
+3+h| Complex types
+
+| <<object, `object`>>         | `struct`    | 0
+| <<nested, `nested`>>         | `struct`    | 0
+
+3+h| Unsupported types
+
+| _types not mentioned above_  | `unsupported`| 0
+
+|===
+
+Obviously, not all types in {es} have an equivalent in SQL and vice-versa hence why, {es-sql}
+uses the data type _particularities_ of the former over the latter as ultimately {es} is the backing store.
+
+
+[[sql-multi-field]]
+[float]
+==== SQL and multi-fields
+
+A core concept in {es} is that of an `analyzed` field, that is a full-text value that is interpreted in order
+to be effectively indexed. These fields are of type <<text, `text`>> and are not used for sorting or aggregations as their actual value depends on the <<analyzer, `analyzer`>> used hence why {es} also offers the <<keyword, `keyword`>> type for storing the _exact_
+value.
+
+In most case, and the default actually, is to use both types when for strings which {es} supports through <<multi-fields, multi fields>>, that is the ability to index the same string in multiple ways; for example index it both as `text` for search but also as `keyword` for sorting and aggregations.
+
+As SQL requires exact values, when encountering a `text` field {es-sql} will search for an exact multi-field that it can use for comparisons, sorting and aggregations.
+To do that, it will search for the first `keyword` that it can find that is _not_ normalized and use that as the original field _exact_ value.
+
+Consider the following `string` mapping:
+
+[source, js]
+----
+{
+    "first_name" : {
+        "type" : "text",
+        "fields" : {
+            "raw" : {
+                "type" : "keyword"
+            }
+        }
+    }
+}
+----
+
+The following SQL query:
+
+[source, sql]
+----
+SELECT first_name FROM index WHERE first_name = 'John'
+----
+
+is identical to:
+
+[source, sql]
+----
+SELECT first_name FROM index WHERE first_name.raw = 'John'
+----
+
+as {es-sql} automatically _picks_ up the `raw` multi-field from `raw` for exact matching.
 
-// TODO finish this
-List of data types in SQL and how they actually map to Elasticsearch.
-Also mention the corner cases - multi-fields, names with dots, etc...

x-pack/docs/en/sql/overview.asciidoc

@@ -0,0 +1,30 @@
+[[sql-overview]]
+== Overview
+
+{es-sql} aims to provide a powerful yet lightweight SQL interface to {es}.
+
+[[sql-introduction]]
+=== Introduction
+
+{es-sql} is an X-Pack component that allows SQL-like queries to be executed in real-time against {es}.
+Whether using the REST interface, command-line or JDBC, any client can use SQL to search and aggregate data
+_natively_ inside {es}.
+One can think of {es-sql} as a _translator_, one that understands both SQL and {es} and makes it easy to read and process data in real-time, at scale by leveraging {es} capabilities.
+
+[[sql-why]]
+=== Why {es-sql} ?
+
+Native integration::
+
+{es-sql} is built from the ground up for {es}. Each and every query is efficiently executed against the relevant nodes according to the underlying storage.
+
+No external parts::
+
+No need for additional hardware, processes, runtimes or libraries to query {es}; {es-sql} eliminates extra moving parts by running _inside_ the {es} cluster.
+
+Lightweight and efficient::
+
+{es-sql} does not abstract {es} and its search capabilities - on the contrary, it embrases and exposes to SQL to allow proper full-text search, in real-time, in the same declarative, succint fashion.
+
+
+

x-pack/docs/en/sql/standalone.asciidoc

@@ -1,9 +0,0 @@
-[[elasticsearch-sql-standalone]]
-= Elasticsearch SQL Standalone
-
-:es-repo-dir:           {docdir}/../../../../../elasticsearch/docs
-
-:edit_url:
-include::{es-repo-dir}/reference/index-shared3.asciidoc[]
-:edit_url!:
-include::index.asciidoc[]

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java

@@ -145,7 +145,7 @@ public final class SSLChannelContext extends SocketChannelContext {
                 selector.queueWrite(writeOperation);
                 return;
             }
-            selector.queueWriteInChannelBuffer(writeOperation);
+            selector.writeToChannel(writeOperation);
         }
     }
 

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContextTests.java

@@ -345,7 +345,7 @@ public class SSLChannelContextTests extends ESTestCase {
         context.closeChannel();
 
         ArgumentCaptor<WriteOperation> captor = ArgumentCaptor.forClass(WriteOperation.class);
-        verify(selector).queueWriteInChannelBuffer(captor.capture());
+        verify(selector).writeToChannel(captor.capture());
 
         context.queueWriteOperation(captor.getValue());
         verify(sslDriver).initiateClose();