security: REST spec cleanup and authenticate tests create user

This change cleans up the rest API specs and changes the documentation field to a link. Additionally,
the integration tests for xpack now use the elastic user rather than a dummy user from a file realm.

Closes elastic/elasticsearch#2458
Closes elastic/elasticsearch#2437

Original commit: elastic/x-pack-elasticsearch@8059a0d856

elasticsearch/x-pack/build.gradle

@@ -1,6 +1,8 @@
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.test.NodeInfo
 
+import java.nio.charset.StandardCharsets
+
 group 'org.elasticsearch.plugin'
 
 apply plugin: 'elasticsearch.esplugin'
@@ -132,15 +134,33 @@ integTest {
   systemProperty 'tests.rest.blacklist', 'getting_started/10_monitor_cluster_health/*,bulk/10_basic/*'
   cluster {
     setting 'xpack.monitoring.agent.interval', '3s'
-    setupCommand 'setupDummyUser', 'bin/x-pack/users', 'useradd', 'test_user', '-p', 'changeme', '-r', 'superuser'
     waitCondition = { NodeInfo node, AntBuilder ant ->
         File tmpFile = new File(node.cwd, 'wait.success')
-        ant.get(src: "http://${node.httpUri()}",
+        for (int i = 0; i < 10; i++) {
-                dest: tmpFile.toString(),
+          // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned
-                username: "test_user",
+          HttpURLConnection httpURLConnection = null;
-                password: "changeme",
+          try {
-                ignoreerrors: true, // do not fail on error, so logging buffers can be flushed by the wait task
+            httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}").openConnection();
-                retries: 10)
+            httpURLConnection.setRequestProperty("Authorization", "Basic " +
+                    Base64.getEncoder().encodeToString("elastic:changeme".getBytes(StandardCharsets.UTF_8)));
+            httpURLConnection.setRequestMethod("GET");
+            httpURLConnection.connect();
+            if (httpURLConnection.getResponseCode() == 200) {
+              tmpFile.withWriter StandardCharsets.UTF_8.name(), {
+                it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
+              }
+            }
+          } catch (Exception e) {
+            e.printStackTrace()
+          } finally {
+            if (httpURLConnection != null) {
+              httpURLConnection.disconnect();
+            }
+          }
+
+          // did not start, so wait a bit before trying again
+          Thread.sleep(500L);
+        }
         return tmpFile.exists()
     }
   }

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalAuthenticationService.java

@@ -211,7 +211,7 @@ public class InternalAuthenticationService extends AbstractComponent implements
                     }
                 }
             } catch (Exception e) {
-                logger.debug("authentication failed for principal [{}]", e, request);
+                logger.debug("authentication failed for principal [{}], [{}] ", e, token.principal(), request);
                 throw request.exceptionProcessingRequest(e, token);
             } finally {
                 token.clearCredentials();

elasticsearch/x-pack/security/src/test/resources/rest-api-spec/api/xpack.security.authenticate.json

@@ -1,6 +1,6 @@
 {
   "xpack.security.authenticate": {
-    "documentation": "Retrieve details about the currently authenticated user",
+    "documentation": "https://www.elastic.co/guide/en/x-pack/master/security-api-authenticate.html",
     "methods": [ "GET" ],
     "url": {
       "path": "/_xpack/security/_authenticate",

elasticsearch/x-pack/security/src/test/resources/rest-api-spec/api/xpack.security.change_password.json

@@ -1,6 +1,6 @@
 {
   "xpack.security.change_password": {
-    "documentation": "Change the password of a user",
+    "documentation": "https://www.elastic.co/guide/en/x-pack/master/security-api-change-password.html",
     "methods": [ "PUT", "POST" ],
     "url": {
       "path": "/_xpack/security/user/{username}/_password",

elasticsearch/x-pack/security/src/test/resources/rest-api-spec/api/xpack.security.clear_cached_realms.json

@@ -1,20 +1,20 @@
 {
   "xpack.security.clear_cached_realms": {
-    "documentation": "Clears the internal user caches for specified realms",
+    "documentation": "https://www.elastic.co/guide/en/x-pack/current/security-api-clear-cache.html",
     "methods": [ "POST" ],
     "url": {
       "path": "/_xpack/security/realm/{realms}/_clear_cache",
       "paths": [ "/_xpack/security/realm/{realms}/_clear_cache" ],
       "parts": {
         "realms": {
-          "type" : "string",
+          "type" : "list",
           "description" : "Comma-separated list of realms to clear",
           "required" : true
         }
       },
       "params": {
         "usernames": {
-          "type" : "string",
+          "type" : "list",
           "description" : "Comma-separated list of usernames to clear from the cache",
           "required" : false
         }

elasticsearch/x-pack/security/src/test/resources/rest-api-spec/api/xpack.security.clear_cached_roles.json

@@ -1,13 +1,13 @@
 {
   "xpack.security.clear_cached_roles": {
-    "documentation": "Clears the internal caches for specified roles",
+    "documentation": "https://www.elastic.co/guide/en/x-pack/master/security-api-roles.html#security-api-clear-role-cache",
-    "methods": [ "PUT", "POST" ],
+    "methods": [ "POST" ],
     "url": {
       "path": "/_xpack/security/role/{name}/_clear_cache",
       "paths": [ "/_xpack/security/role/{name}/_clear_cache" ],
       "parts": {
         "name": {
-          "type" : "string",
+          "type" : "list",
           "description" : "Role name",
           "required" : true
         }

elasticsearch/x-pack/security/src/test/resources/rest-api-spec/api/xpack.security.delete_role.json

@@ -1,6 +1,6 @@
 {
   "xpack.security.delete_role": {
-    "documentation": "Remove a role from the native realm",
+    "documentation": "https://www.elastic.co/guide/en/x-pack/master/security-api-roles.html#security-api-delete-role",
     "methods": [ "DELETE" ],
     "url": {
       "path": "/_xpack/security/role/{name}",

elasticsearch/x-pack/security/src/test/resources/rest-api-spec/api/xpack.security.delete_user.json

@@ -1,6 +1,6 @@
 {
   "xpack.security.delete_user": {
-    "documentation": "Remove a user from the native realm",
+    "documentation": "https://www.elastic.co/guide/en/x-pack/master/security-api-users.html#security-api-delete-user",
     "methods": [ "DELETE" ],
     "url": {
       "path": "/_xpack/security/user/{username}",

elasticsearch/x-pack/security/src/test/resources/rest-api-spec/api/xpack.security.get_role.json

@@ -1,6 +1,6 @@
 {
   "xpack.security.get_role": {
-    "documentation": "Retrieve one or more roles from the native realm",
+    "documentation": "https://www.elastic.co/guide/en/x-pack/master/security-api-roles.html#security-api-get-role",
     "methods": [ "GET" ],
     "url": {
       "path": "/_xpack/security/role/{name}",

elasticsearch/x-pack/security/src/test/resources/rest-api-spec/api/xpack.security.get_user.json

@@ -1,6 +1,6 @@
 {
   "xpack.security.get_user": {
-    "documentation": "Retrieve one or more users from the native realm",
+    "documentation": "https://www.elastic.co/guide/en/x-pack/master/security-api-users.html#security-api-get-user",
     "methods": [ "GET" ],
     "url": {
       "path": "/_xpack/security/user/{username}",

elasticsearch/x-pack/security/src/test/resources/rest-api-spec/api/xpack.security.put_role.json

@@ -1,6 +1,6 @@
 {
   "xpack.security.put_role": {
-    "documentation": "Update or create a role for the native realm",
+    "documentation": "https://www.elastic.co/guide/en/x-pack/master/security-api-roles.html#security-api-put-role",
     "methods": [ "PUT", "POST" ],
     "url": {
       "path": "/_xpack/security/role/{name}",

elasticsearch/x-pack/security/src/test/resources/rest-api-spec/api/xpack.security.put_user.json

@@ -1,6 +1,6 @@
 {
   "xpack.security.put_user": {
-    "documentation": "Update or create a user for the native realm",
+    "documentation": "https://www.elastic.co/guide/en/x-pack/master/security-api-users.html#security-api-put-user",
     "methods": [ "PUT", "POST" ],
     "url": {
       "path": "/_xpack/security/user/{username}",

elasticsearch/x-pack/security/src/test/resources/rest-api-spec/test/authenticate/10_basic.yaml

@@ -1,11 +1,29 @@
----
+setup:
-"Test authenticate api":
+  - skip:
+      features: headers
 
   - do:
       cluster.health:
           wait_for_status: yellow
+
   - do:
+      xpack.security.put_user:
+        username: "authenticate_user"
+        body:  >
+          {
+            "password" : "changeme",
+            "roles" : [ "superuser" ],
+            "full_name" : "Authenticate User"
+          }
+
+---
+"Test authenticate api":
+
+  - do:
+      headers:
+        Authorization: "Basic YXV0aGVudGljYXRlX3VzZXI6Y2hhbmdlbWU="
       xpack.security.authenticate: {}
 
-  - match: { username: "test_user" }
+  - match: { username: "authenticate_user" }
   - match: { roles.0:  "superuser" }
+  - match: { full_name: "Authenticate User" }

elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestCase.java

@@ -39,7 +39,7 @@ import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordTok
 
 public abstract class XPackRestTestCase extends ESRestTestCase {
 
-    private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("test_user", new SecuredString("changeme".toCharArray()));
+    private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("elastic", new SecuredString("changeme".toCharArray()));
 
     public XPackRestTestCase(@Name("yaml") RestTestCandidate testCandidate) {
         super(testCandidate);