Security tests for SQL's CLI and JDBC (elastic/x-pack-elasticsearch#2770)
Add security tests for SQL's CLI and JDBC features. I do this by factoring out all the "actions" from the existing REST tests into an interface and implement it for REST, CLI, and JDBC. This way we can share the same audit log assertions across tests and we can be sure that the REST, CLI, and JDBC tests cover all the same use cases. Original commit: elastic/x-pack-elasticsearch@82ff66a520
This commit is contained in:
Nik Everett
GitHub
parent
65f2b9fe01
commit
56ce29c6bf
|
|
@ -16,6 +16,6 @@ public class CliErrorsIT extends ErrorsTestCase {
|
|||
|
||||
@Override
|
||||
protected String esUrlPrefix() {
|
||||
return CliFetchSizeIT.securityEsUrlPrefix();
|
||||
return CliSecurityIT.adminEsUrlPrefix();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,10 +9,6 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.xpack.qa.sql.cli.FetchSizeTestCase;
|
||||
|
||||
public class CliFetchSizeIT extends FetchSizeTestCase {
|
||||
static String securityEsUrlPrefix() {
|
||||
return "test_admin:x-pack-test-password@";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Settings restClientSettings() {
|
||||
return RestSqlIT.securitySettings();
|
||||
|
|
@ -20,6 +16,6 @@ public class CliFetchSizeIT extends FetchSizeTestCase {
|
|||
|
||||
@Override
|
||||
protected String esUrlPrefix() {
|
||||
return securityEsUrlPrefix();
|
||||
return CliSecurityIT.adminEsUrlPrefix();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,119 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
package org.elasticsearch.xpack.qa.sql.security;
|
||||
|
||||
import org.elasticsearch.xpack.qa.sql.cli.RemoteCli;
|
||||
|
||||
import static org.elasticsearch.xpack.qa.sql.cli.CliIntegrationTestCase.elasticsearchAddress;
|
||||
import static org.hamcrest.Matchers.both;
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
import static org.hamcrest.Matchers.startsWith;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class CliSecurityIT extends SqlSecurityTestCase {
|
||||
static String adminEsUrlPrefix() {
|
||||
return "test_admin:x-pack-test-password@";
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform security test actions using the CLI.
|
||||
*/
|
||||
private static class CliActions implements Actions {
|
||||
@Override
|
||||
public void queryWorksAsAdmin() throws Exception {
|
||||
try (RemoteCli cli = new RemoteCli(adminEsUrlPrefix() + elasticsearchAddress())) {
|
||||
assertThat(cli.command("SELECT * FROM test ORDER BY a"), containsString("a | b | c"));
|
||||
assertEquals("---------------+---------------+---------------", cli.readLine());
|
||||
assertThat(cli.readLine(), containsString("1 |2 |3"));
|
||||
assertThat(cli.readLine(), containsString("4 |5 |6"));
|
||||
assertEquals("[0m", cli.readLine());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void expectMatchesAdmin(String adminSql, String user, String userSql) throws Exception {
|
||||
List<String> adminResult = new ArrayList<>();
|
||||
try (RemoteCli cli = new RemoteCli(adminEsUrlPrefix() + elasticsearchAddress())) {
|
||||
adminResult.add(cli.command(adminSql));
|
||||
String line;
|
||||
do {
|
||||
line = cli.readLine();
|
||||
adminResult.add(line);
|
||||
} while (false == line.equals("[0m"));
|
||||
adminResult.add(line);
|
||||
}
|
||||
|
||||
Iterator<String> expected = adminResult.iterator();
|
||||
try (RemoteCli cli = new RemoteCli(userPrefix(user) + elasticsearchAddress())) {
|
||||
assertTrue(expected.hasNext());
|
||||
assertEquals(expected.next(), cli.command(userSql));
|
||||
String line;
|
||||
do {
|
||||
line = cli.readLine();
|
||||
assertTrue(expected.hasNext());
|
||||
assertEquals(expected.next(), line);
|
||||
} while (false == line.equals("[0m"));
|
||||
assertTrue(expected.hasNext());
|
||||
assertEquals(expected.next(), line);
|
||||
assertFalse(expected.hasNext());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void expectDescribe(Map<String, String> columns, String user) throws Exception {
|
||||
try (RemoteCli cli = new RemoteCli(userPrefix(user) + elasticsearchAddress())) {
|
||||
assertThat(cli.command("DESCRIBE test"), containsString("column | type"));
|
||||
assertEquals("---------------+---------------", cli.readLine());
|
||||
for (Map.Entry<String, String> column : columns.entrySet()) {
|
||||
assertThat(cli.readLine(), both(startsWith(column.getKey())).and(containsString("|" + column.getValue())));
|
||||
}
|
||||
assertEquals("[0m", cli.readLine());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void expectShowTables(List<String> tables, String user) throws Exception {
|
||||
try (RemoteCli cli = new RemoteCli(userPrefix(user) + elasticsearchAddress())) {
|
||||
assertThat(cli.command("SHOW TABLES"), containsString("table"));
|
||||
assertEquals("---------------", cli.readLine());
|
||||
for (String table : tables) {
|
||||
assertThat(cli.readLine(), containsString(table));
|
||||
}
|
||||
assertEquals("[0m", cli.readLine());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void expectForbidden(String user, String sql) throws Exception {
|
||||
try (RemoteCli cli = new RemoteCli(userPrefix(user) + elasticsearchAddress())) {
|
||||
assertThat(cli.command(sql), containsString("is unauthorized for user [" + user + "]"));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void expectUnknownColumn(String user, String sql, String column) throws Exception {
|
||||
try (RemoteCli cli = new RemoteCli(userPrefix(user) + elasticsearchAddress())) {
|
||||
assertThat(cli.command(sql), containsString("[1;31mBad request"));
|
||||
assertThat(cli.readLine(), containsString("Unknown column [" + column + "][1;23;31m][0m"));
|
||||
}
|
||||
}
|
||||
|
||||
private String userPrefix(String user) {
|
||||
if (user == null) {
|
||||
return adminEsUrlPrefix();
|
||||
}
|
||||
return user + ":testpass@";
|
||||
}
|
||||
}
|
||||
|
||||
public CliSecurityIT() {
|
||||
super(new CliActions());
|
||||
}
|
||||
}
|
||||
|
|
@ -16,6 +16,6 @@ public class CliSelectIT extends SelectTestCase {
|
|||
|
||||
@Override
|
||||
protected String esUrlPrefix() {
|
||||
return CliFetchSizeIT.securityEsUrlPrefix();
|
||||
return CliSecurityIT.adminEsUrlPrefix();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,6 @@ public class CliShowIT extends ShowTestCase {
|
|||
|
||||
@Override
|
||||
protected String esUrlPrefix() {
|
||||
return CliFetchSizeIT.securityEsUrlPrefix();
|
||||
return CliSecurityIT.adminEsUrlPrefix();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,13 +11,6 @@ import org.elasticsearch.xpack.qa.sql.jdbc.ConnectionTestCase;
|
|||
import java.util.Properties;
|
||||
|
||||
public class JdbcConnectionIT extends ConnectionTestCase {
|
||||
static Properties securityProperties() {
|
||||
Properties prop = new Properties();
|
||||
prop.put("user", "test_admin");
|
||||
prop.put("pass", "x-pack-test-password");
|
||||
return prop;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Settings restClientSettings() {
|
||||
return RestSqlIT.securitySettings();
|
||||
|
|
@ -25,6 +18,6 @@ public class JdbcConnectionIT extends ConnectionTestCase {
|
|||
|
||||
@Override
|
||||
protected Properties connectionProperties() {
|
||||
return securityProperties();
|
||||
return JdbcSecurityIT.adminProperties();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,6 @@ public class JdbcCsvSpecIT extends CsvSpecTestCase {
|
|||
|
||||
@Override
|
||||
protected Properties connectionProperties() {
|
||||
return JdbcConnectionIT.securityProperties();
|
||||
return JdbcSecurityIT.adminProperties();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,6 @@ public class JdbcErrorsIT extends ErrorsTestCase {
|
|||
|
||||
@Override
|
||||
protected Properties connectionProperties() {
|
||||
return JdbcConnectionIT.securityProperties();
|
||||
return JdbcSecurityIT.adminProperties();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,6 @@ public class JdbcFetchSizeIT extends FetchSizeTestCase {
|
|||
|
||||
@Override
|
||||
protected Properties connectionProperties() {
|
||||
return JdbcConnectionIT.securityProperties();
|
||||
return JdbcSecurityIT.adminProperties();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,132 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
package org.elasticsearch.xpack.qa.sql.security;
|
||||
|
||||
import org.elasticsearch.xpack.qa.sql.jdbc.LocalH2;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.StringJoiner;
|
||||
|
||||
import static org.elasticsearch.xpack.qa.sql.jdbc.JdbcAssert.assertResultSets;
|
||||
import static org.elasticsearch.xpack.qa.sql.jdbc.JdbcIntegrationTestCase.elasticsearchAddress;
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
|
||||
public class JdbcSecurityIT extends SqlSecurityTestCase {
|
||||
static Properties adminProperties() {
|
||||
Properties prop = new Properties();
|
||||
prop.put("user", "test_admin");
|
||||
prop.put("pass", "x-pack-test-password");
|
||||
return prop;
|
||||
}
|
||||
|
||||
private static class JdbcActions implements Actions {
|
||||
@Override
|
||||
public void queryWorksAsAdmin() throws Exception {
|
||||
try (Connection h2 = LocalH2.anonymousDb();
|
||||
Connection es = DriverManager.getConnection(elasticsearchAddress(), adminProperties())) {
|
||||
h2.createStatement().executeUpdate("CREATE TABLE test (a BIGINT, b BIGINT, c BIGINT)");
|
||||
h2.createStatement().executeUpdate("INSERT INTO test (a, b, c) VALUES (1, 2, 3), (4, 5, 6)");
|
||||
|
||||
ResultSet expected = h2.createStatement().executeQuery("SELECT * FROM test ORDER BY a");
|
||||
assertResultSets(expected, es.createStatement().executeQuery("SELECT * FROM test ORDER BY a"));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void expectMatchesAdmin(String adminSql, String user, String userSql) throws Exception {
|
||||
try (Connection admin = DriverManager.getConnection(elasticsearchAddress(), adminProperties());
|
||||
Connection other = DriverManager.getConnection(elasticsearchAddress(), userProperties(user))) {
|
||||
ResultSet expected = admin.createStatement().executeQuery(adminSql);
|
||||
assertResultSets(expected, other.createStatement().executeQuery(userSql));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void expectDescribe(Map<String, String> columns, String user) throws Exception {
|
||||
try (Connection h2 = LocalH2.anonymousDb();
|
||||
Connection es = DriverManager.getConnection(elasticsearchAddress(), userProperties(user))) {
|
||||
// h2 doesn't have the same sort of DESCRIBE that we have so we emulate it
|
||||
h2.createStatement().executeUpdate("CREATE TABLE mock (column VARCHAR, type VARCHAR)");
|
||||
StringBuilder insert = new StringBuilder();
|
||||
insert.append("INSERT INTO mock (column, type) VALUES ");
|
||||
boolean first = true;
|
||||
for (Map.Entry<String, String> column : columns.entrySet()) {
|
||||
if (first) {
|
||||
first = false;
|
||||
} else {
|
||||
insert.append(", ");
|
||||
}
|
||||
insert.append("('").append(column.getKey()).append("', '").append(column.getValue()).append("')");
|
||||
}
|
||||
h2.createStatement().executeUpdate(insert.toString());
|
||||
|
||||
ResultSet expected = h2.createStatement().executeQuery("SELECT * FROM mock");
|
||||
assertResultSets(expected, es.createStatement().executeQuery("DESCRIBE test"));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void expectShowTables(List<String> tables, String user) throws Exception {
|
||||
try (Connection h2 = LocalH2.anonymousDb();
|
||||
Connection es = DriverManager.getConnection(elasticsearchAddress(), userProperties(user))) {
|
||||
// h2 doesn't spit out the same columns we do so we emulate
|
||||
h2.createStatement().executeUpdate("CREATE TABLE mock (table VARCHAR)");
|
||||
StringBuilder insert = new StringBuilder();
|
||||
insert.append("INSERT INTO mock (table) VALUES ");
|
||||
boolean first = true;
|
||||
for (String table : tables) {
|
||||
if (first) {
|
||||
first = false;
|
||||
} else {
|
||||
insert.append(", ");
|
||||
}
|
||||
insert.append("('").append(table).append("')");
|
||||
}
|
||||
h2.createStatement().executeUpdate(insert.toString());
|
||||
|
||||
ResultSet expected = h2.createStatement().executeQuery("SELECT * FROM mock ORDER BY table");
|
||||
assertResultSets(expected, es.createStatement().executeQuery("SHOW TABLES"));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void expectForbidden(String user, String sql) throws Exception {
|
||||
SQLException e;
|
||||
try (Connection connection = DriverManager.getConnection(elasticsearchAddress(), userProperties(user))) {
|
||||
e = expectThrows(SQLException.class, () -> connection.createStatement().executeQuery(sql));
|
||||
}
|
||||
assertThat(e.getMessage(), containsString("is unauthorized for user [" + user + "]"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void expectUnknownColumn(String user, String sql, String column) throws Exception {
|
||||
SQLException e;
|
||||
try (Connection connection = DriverManager.getConnection(elasticsearchAddress(), userProperties(user))) {
|
||||
e = expectThrows(SQLException.class, () -> connection.createStatement().executeQuery(sql));
|
||||
}
|
||||
assertThat(e.getMessage(), containsString("Unknown column [" + column + "]"));
|
||||
}
|
||||
|
||||
private Properties userProperties(String user) {
|
||||
if (user == null) {
|
||||
return adminProperties();
|
||||
}
|
||||
Properties prop = new Properties();
|
||||
prop.put("user", user);
|
||||
prop.put("pass", "testpass");
|
||||
return prop;
|
||||
}
|
||||
}
|
||||
|
||||
public JdbcSecurityIT() {
|
||||
super(new JdbcActions());
|
||||
}
|
||||
}
|
||||
|
|
@ -18,6 +18,6 @@ public class JdbcShowTablesIT extends ShowTablesTestCase {
|
|||
|
||||
@Override
|
||||
protected Properties connectionProperties() {
|
||||
return JdbcConnectionIT.securityProperties();
|
||||
return JdbcSecurityIT.adminProperties();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,6 @@ public class JdbcSqlSpecIT extends SqlSpecTestCase {
|
|||
|
||||
@Override
|
||||
protected Properties connectionProperties() {
|
||||
return JdbcConnectionIT.securityProperties();
|
||||
return JdbcSecurityIT.adminProperties();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,510 +9,112 @@ import org.apache.http.Header;
|
|||
import org.apache.http.entity.ContentType;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.message.BasicHeader;
|
||||
import org.elasticsearch.SpecialPermission;
|
||||
import org.elasticsearch.client.Response;
|
||||
import org.elasticsearch.client.ResponseException;
|
||||
import org.elasticsearch.common.CheckedFunction;
|
||||
import org.elasticsearch.common.Nullable;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.io.PathUtils;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentHelper;
|
||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||
import org.elasticsearch.test.NotEqualMessageBuilder;
|
||||
import org.elasticsearch.test.rest.ESRestTestCase;
|
||||
import org.hamcrest.Matcher;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.AccessController;
|
||||
import java.security.PrivilegedAction;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static java.util.Collections.emptyList;
|
||||
import static org.elasticsearch.xpack.qa.sql.rest.RestSqlTestCase.columnInfo;
|
||||
import static java.util.Collections.emptyMap;
|
||||
import static java.util.Collections.singletonList;
|
||||
import static java.util.Collections.singletonMap;
|
||||
import static org.elasticsearch.xpack.qa.sql.rest.RestSqlTestCase.columnInfo;
|
||||
import static org.hamcrest.Matchers.contains;
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
import static org.hamcrest.Matchers.empty;
|
||||
import static org.hamcrest.Matchers.hasItems;
|
||||
|
||||
public class RestSqlSecurityIT extends ESRestTestCase {
|
||||
private static final String SQL_ACTION_NAME = "indices:data/read/sql";
|
||||
private static final String SQL_INDICES_ACTION_NAME = "indices:data/read/sql/tables";
|
||||
/**
|
||||
* Location of the audit log file. We could technically figure this out by reading the admin
|
||||
* APIs but it isn't worth doing because we also have to give ourselves permission to read
|
||||
* the file and that must be done by setting a system property and reading it in
|
||||
* {@code plugin-security.policy}. So we may as well have gradle set the property.
|
||||
*/
|
||||
private static final Path AUDIT_LOG_FILE;
|
||||
static {
|
||||
String auditLogFileString = System.getProperty("tests.audit.logfile");
|
||||
if (null == auditLogFileString) {
|
||||
throw new IllegalStateException("tests.audit.logfile must be set to run this test. It is automatically "
|
||||
+ "set by gradle. If you must set it yourself then it should be the absolute path to the audit "
|
||||
+ "log file generated by running x-pack with audit logging enabled.");
|
||||
public class RestSqlSecurityIT extends SqlSecurityTestCase {
|
||||
private static class RestActions implements Actions {
|
||||
@Override
|
||||
public void queryWorksAsAdmin() throws Exception {
|
||||
Map<String, Object> expected = new HashMap<>();
|
||||
expected.put("columns", Arrays.asList(
|
||||
columnInfo("a", "long"),
|
||||
columnInfo("b", "long"),
|
||||
columnInfo("c", "long")));
|
||||
expected.put("rows", Arrays.asList(
|
||||
Arrays.asList(1, 2, 3),
|
||||
Arrays.asList(4, 5, 6)));
|
||||
expected.put("size", 2);
|
||||
|
||||
assertResponse(expected, runSql(null, "SELECT * FROM test ORDER BY a"));
|
||||
}
|
||||
AUDIT_LOG_FILE = PathUtils.get(auditLogFileString);
|
||||
}
|
||||
|
||||
private static boolean oneTimeSetup = false;
|
||||
private static boolean auditFailure = false;
|
||||
|
||||
/**
|
||||
* How much of the audit log was written before the test started.
|
||||
*/
|
||||
private long auditLogWrittenBeforeTestStart;
|
||||
|
||||
/**
|
||||
* All tests run as a an administrative user but use
|
||||
* <code>es-security-runas-user</code> to become a less privileged user when needed.
|
||||
*/
|
||||
@Override
|
||||
protected Settings restClientSettings() {
|
||||
return RestSqlIT.securitySettings();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean preserveIndicesUponCompletion() {
|
||||
/* We can't wipe the cluster between tests because that nukes the audit
|
||||
* trail index which makes the auditing flaky. Instead we wipe all
|
||||
* indices after the entire class is finished. */
|
||||
return true;
|
||||
}
|
||||
|
||||
@Before
|
||||
public void oneTimeSetup() throws Exception {
|
||||
if (oneTimeSetup) {
|
||||
/* Since we don't wipe the cluster between tests we only need to
|
||||
* write the test data once. */
|
||||
return;
|
||||
@Override
|
||||
public void expectMatchesAdmin(String adminSql, String user, String userSql) throws Exception {
|
||||
assertResponse(runSql(null, adminSql), runSql(user, userSql));
|
||||
}
|
||||
StringBuilder bulk = new StringBuilder();
|
||||
bulk.append("{\"index\":{\"_index\": \"test\", \"_type\": \"doc\", \"_id\":\"1\"}\n");
|
||||
bulk.append("{\"a\": 1, \"b\": 2, \"c\": 3}\n");
|
||||
bulk.append("{\"index\":{\"_index\": \"test\", \"_type\": \"doc\", \"_id\":\"2\"}\n");
|
||||
bulk.append("{\"a\": 4, \"b\": 5, \"c\": 6}\n");
|
||||
bulk.append("{\"index\":{\"_index\": \"bort\", \"_type\": \"doc\", \"_id\":\"1\"}\n");
|
||||
bulk.append("{\"a\": \"test\"}\n");
|
||||
client().performRequest("PUT", "/_bulk", singletonMap("refresh", "true"),
|
||||
new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON));
|
||||
oneTimeSetup = true;
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setInitialAuditLogOffset() throws IOException {
|
||||
SecurityManager sm = System.getSecurityManager();
|
||||
if (sm != null) {
|
||||
sm.checkPermission(new SpecialPermission());
|
||||
@Override
|
||||
public void expectDescribe(Map<String, String> columns, String user) throws Exception {
|
||||
Map<String, Object> expected = new HashMap<>(3);
|
||||
expected.put("columns", Arrays.asList(
|
||||
columnInfo("column", "keyword"),
|
||||
columnInfo("type", "keyword")));
|
||||
List<List<String>> rows = new ArrayList<>(columns.size());
|
||||
for (Map.Entry<String, String> column : columns.entrySet()) {
|
||||
rows.add(Arrays.asList(column.getKey(), column.getValue()));
|
||||
}
|
||||
expected.put("rows", rows);
|
||||
expected.put("size", columns.size());
|
||||
|
||||
assertResponse(expected, runSql(user, "DESCRIBE test"));
|
||||
}
|
||||
AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
|
||||
if (false == Files.exists(AUDIT_LOG_FILE)) {
|
||||
auditLogWrittenBeforeTestStart = 0;
|
||||
return null;
|
||||
}
|
||||
if (false == Files.isRegularFile(AUDIT_LOG_FILE)) {
|
||||
throw new IllegalStateException("expected tests.audit.logfile [" + AUDIT_LOG_FILE + "]to be a plain file but wasn't");
|
||||
}
|
||||
try {
|
||||
auditLogWrittenBeforeTestStart = Files.size(AUDIT_LOG_FILE);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void wipeIndicesAfterTests() throws IOException {
|
||||
try {
|
||||
adminClient().performRequest("DELETE", "*");
|
||||
} catch (ResponseException e) {
|
||||
// 404 here just means we had no indexes
|
||||
if (e.getResponse().getStatusLine().getStatusCode() != 404) {
|
||||
throw e;
|
||||
@Override
|
||||
public void expectShowTables(List<String> tables, String user) throws Exception {
|
||||
Map<String, Object> expected = new HashMap<>();
|
||||
expected.put("columns", singletonList(columnInfo("table", "keyword")));
|
||||
List<List<String>> rows = new ArrayList<>();
|
||||
for (String table : tables) {
|
||||
rows.add(singletonList(table));
|
||||
}
|
||||
expected.put("rows", rows);
|
||||
expected.put("size", tables.size());
|
||||
assertResponse(expected, runSql(user, "SHOW TABLES"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void expectForbidden(String user, String sql) {
|
||||
ResponseException e = expectThrows(ResponseException.class, () -> runSql(user, sql));
|
||||
assertThat(e.getMessage(), containsString("403 Forbidden"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void expectUnknownColumn(String user, String sql, String column) throws Exception {
|
||||
ResponseException e = expectThrows(ResponseException.class, () -> runSql(user, sql));
|
||||
assertThat(e.getMessage(), containsString("Unknown column [" + column + "]"));
|
||||
}
|
||||
|
||||
private static Map<String, Object> runSql(@Nullable String asUser, String sql) throws IOException {
|
||||
Header[] headers = asUser == null ? new Header[0] : new Header[] {new BasicHeader("es-security-runas-user", asUser)};
|
||||
Response response = client().performRequest("POST", "/_sql", emptyMap(),
|
||||
new StringEntity("{\"query\": \"" + sql + "\"}", ContentType.APPLICATION_JSON),
|
||||
headers);
|
||||
return toMap(response);
|
||||
}
|
||||
|
||||
private static void assertResponse(Map<String, Object> expected, Map<String, Object> actual) {
|
||||
if (false == expected.equals(actual)) {
|
||||
NotEqualMessageBuilder message = new NotEqualMessageBuilder();
|
||||
message.compareMaps(actual, expected);
|
||||
fail("Response does not match:\n" + message.toString());
|
||||
}
|
||||
}
|
||||
|
||||
private static Map<String, Object> toMap(Response response) throws IOException {
|
||||
try (InputStream content = response.getEntity().getContent()) {
|
||||
return XContentHelper.convertToMap(JsonXContent.jsonXContent, content, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NOCOMMIT we're going to need to test jdbc and cli with these too!
|
||||
// NOCOMMIT we'll have to test scrolling as well
|
||||
// NOCOMMIT assert that we don't have more audit logs then what we expect.
|
||||
|
||||
public void testQueryWorksAsAdmin() throws Exception {
|
||||
Map<String, Object> expected = new HashMap<>();
|
||||
expected.put("columns", Arrays.asList(
|
||||
columnInfo("a", "long"),
|
||||
columnInfo("b", "long"),
|
||||
columnInfo("c", "long")));
|
||||
expected.put("rows", Arrays.asList(
|
||||
Arrays.asList(1, 2, 3),
|
||||
Arrays.asList(4, 5, 6)));
|
||||
expected.put("size", 2);
|
||||
assertResponse(expected, runSql("SELECT * FROM test ORDER BY a", null));
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
}
|
||||
|
||||
public void testQueryWithFullAccess() throws Exception {
|
||||
createUser("full_access", "read_all");
|
||||
|
||||
assertResponse(runSql("SELECT * FROM test ORDER BY a", null), runSql("SELECT * FROM test ORDER BY a", "full_access"));
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
assertAuditForSqlGetTableSyncGranted("full_access", "test");
|
||||
}
|
||||
|
||||
public void testQueryNoAccess() throws Exception {
|
||||
createUser("no_access", "read_nothing");
|
||||
|
||||
ResponseException e = expectThrows(ResponseException.class, () -> runSql("SELECT * FROM test", "no_access"));
|
||||
assertThat(e.getMessage(), containsString("403 Forbidden"));
|
||||
assertAuditEvents(audit(false, SQL_ACTION_NAME, "no_access", empty()));
|
||||
}
|
||||
|
||||
public void testQueryWrongAccess() throws Exception {
|
||||
createUser("wrong_access", "read_something_else");
|
||||
|
||||
ResponseException e = expectThrows(ResponseException.class, () -> runSql("SELECT * FROM test", "wrong_access"));
|
||||
assertThat(e.getMessage(), containsString("403 Forbidden"));
|
||||
assertAuditEvents(
|
||||
/* This user has permission to run sql queries so they are
|
||||
* given preliminary authorization. */
|
||||
audit(true, SQL_ACTION_NAME, "wrong_access", empty()),
|
||||
/* But as soon as they attempt to resolve an index that
|
||||
* they don't have access to they get denied. */
|
||||
audit(false, SQL_ACTION_NAME, "wrong_access", hasItems("test")));
|
||||
}
|
||||
|
||||
public void testQuerySingleFieldGranted() throws Exception {
|
||||
createUser("only_a", "read_test_a");
|
||||
|
||||
assertResponse(runSql("SELECT a FROM test", null), runSql("SELECT * FROM test", "only_a"));
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
assertAuditForSqlGetTableSyncGranted("only_a", "test");
|
||||
// clearAuditEvents(); NOCOMMIT
|
||||
expectBadRequest(() -> runSql("SELECT c FROM test", "only_a"), containsString("line 1:8: Unknown column [c]"));
|
||||
/* The user has permission to query the index but one of the
|
||||
* columns that they explicitly mention is hidden from them
|
||||
* by field level access control. This *looks* like a successful
|
||||
* query from the audit side because all the permissions checked
|
||||
* out but it failed in SQL because it couldn't compile the
|
||||
* query without the metadata for the missing field. */
|
||||
assertAuditForSqlGetTableSyncGranted("only_a", "test");
|
||||
}
|
||||
|
||||
public void testQuerySingleFieldExcepted() throws Exception {
|
||||
createUser("not_c", "read_test_a_and_b");
|
||||
|
||||
assertResponse(runSql("SELECT a, b FROM test", null), runSql("SELECT * FROM test", "not_c"));
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
assertAuditForSqlGetTableSyncGranted("not_c", "test");
|
||||
// clearAuditEvents(); NOCOMMIT
|
||||
expectBadRequest(() -> runSql("SELECT c FROM test", "not_c"), containsString("line 1:8: Unknown column [c]"));
|
||||
/* The user has permission to query the index but one of the
|
||||
* columns that they explicitly mention is hidden from them
|
||||
* by field level access control. This *looks* like a successful
|
||||
* query from the audit side because all the permissions checked
|
||||
* out but it failed in SQL because it couldn't compile the
|
||||
* query without the metadata for the missing field. */
|
||||
assertAuditForSqlGetTableSyncGranted("not_c", "test");
|
||||
}
|
||||
|
||||
public void testQueryDocumentExclued() throws Exception {
|
||||
createUser("no_3s", "read_test_without_c_3");
|
||||
|
||||
assertResponse(runSql("SELECT * FROM test WHERE c != 3", null), runSql("SELECT * FROM test", "no_3s"));
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
assertAuditForSqlGetTableSyncGranted("no_3s", "test");
|
||||
}
|
||||
|
||||
public void testShowTablesWorksAsAdmin() throws Exception {
|
||||
Map<String, Object> expected = new HashMap<>();
|
||||
expected.put("columns", singletonList(columnInfo("table", "keyword")));
|
||||
expected.put("rows", Arrays.asList(
|
||||
singletonList("bort"),
|
||||
singletonList("test")));
|
||||
expected.put("size", 2);
|
||||
assertResponse(expected, runSql("SHOW TABLES", null));
|
||||
assertAuditEvents(
|
||||
audit(true, SQL_ACTION_NAME, "test_admin", empty()),
|
||||
audit(true, SQL_INDICES_ACTION_NAME, "test_admin", hasItems("test", "bort")));
|
||||
}
|
||||
|
||||
public void testShowTablesWorksAsFullAccess() throws Exception {
|
||||
createUser("full_access", "read_all");
|
||||
|
||||
assertResponse(runSql("SHOW TABLES", null), runSql("SHOW TABLES", "full_access"));
|
||||
assertAuditEvents(
|
||||
audit(true, SQL_ACTION_NAME, "test_admin", empty()),
|
||||
audit(true, SQL_INDICES_ACTION_NAME, "test_admin", hasItems("test", "bort")),
|
||||
audit(true, SQL_ACTION_NAME, "full_access", empty()),
|
||||
audit(true, SQL_INDICES_ACTION_NAME, "full_access", hasItems("test", "bort")));
|
||||
}
|
||||
|
||||
public void testShowTablesWithNoAccess() throws Exception {
|
||||
createUser("no_access", "read_nothing");
|
||||
|
||||
ResponseException e = expectThrows(ResponseException.class, () -> runSql("SHOW TABLES", "no_access"));
|
||||
assertThat(e.getMessage(), containsString("403 Forbidden"));
|
||||
assertAuditEvents(audit(false, SQL_ACTION_NAME, "no_access", empty()));
|
||||
}
|
||||
|
||||
public void testShowTablesWithLimitedAccess() throws Exception {
|
||||
createUser("read_bort", "read_bort");
|
||||
|
||||
assertResponse(runSql("SHOW TABLES LIKE 'bort'", null), runSql("SHOW TABLES", "read_bort"));
|
||||
assertAuditEvents(
|
||||
audit(true, SQL_ACTION_NAME, "test_admin", empty()),
|
||||
audit(true, SQL_INDICES_ACTION_NAME, "test_admin", contains("bort")),
|
||||
audit(true, SQL_ACTION_NAME, "read_bort", empty()),
|
||||
audit(true, SQL_INDICES_ACTION_NAME, "read_bort", contains("bort")));
|
||||
}
|
||||
|
||||
public void testShowTablesWithLimitedAccessAndPattern() throws Exception {
|
||||
createUser("read_bort", "read_bort");
|
||||
|
||||
Map<String, Object> expected = new HashMap<>();
|
||||
expected.put("columns", singletonList(columnInfo("table", "keyword")));
|
||||
expected.put("rows", emptyList());
|
||||
expected.put("size", 0);
|
||||
|
||||
assertResponse(expected, runSql("SHOW TABLES LIKE 'test'", "read_bort"));
|
||||
assertAuditEvents(
|
||||
audit(true, SQL_ACTION_NAME, "read_bort", empty()),
|
||||
audit(true, SQL_INDICES_ACTION_NAME, "read_bort", contains("*", "-*")));
|
||||
}
|
||||
|
||||
public void testDescribeWorksAsAdmin() throws Exception {
|
||||
Map<String, Object> expected = new HashMap<>();
|
||||
expected.put("columns", Arrays.asList(
|
||||
columnInfo("column", "keyword"),
|
||||
columnInfo("type", "keyword")));
|
||||
expected.put("rows", Arrays.asList(
|
||||
Arrays.asList("a", "BIGINT"),
|
||||
Arrays.asList("b", "BIGINT"),
|
||||
Arrays.asList("c", "BIGINT")));
|
||||
expected.put("size", 3);
|
||||
assertResponse(expected, runSql("DESCRIBE test", null));
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
}
|
||||
|
||||
public void testDescribeWorksAsFullAccess() throws Exception {
|
||||
createUser("full_access", "read_all");
|
||||
|
||||
assertResponse(runSql("DESCRIBE test", null), runSql("DESCRIBE test", "full_access"));
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
assertAuditForSqlGetTableSyncGranted("full_access", "test");
|
||||
}
|
||||
|
||||
public void testDescribeWithNoAccess() throws Exception {
|
||||
createUser("no_access", "read_nothing");
|
||||
|
||||
ResponseException e = expectThrows(ResponseException.class, () -> runSql("DESCRIBE test", "no_access"));
|
||||
assertThat(e.getMessage(), containsString("403 Forbidden"));
|
||||
assertAuditEvents(audit(false, SQL_ACTION_NAME, "no_access", empty()));
|
||||
}
|
||||
|
||||
public void testDescribeWithWrongAccess() throws Exception {
|
||||
createUser("wrong_access", "read_something_else");
|
||||
|
||||
ResponseException e = expectThrows(ResponseException.class, () -> runSql("DESCRIBE test", "wrong_access"));
|
||||
assertThat(e.getMessage(), containsString("403 Forbidden"));
|
||||
assertAuditEvents(
|
||||
/* This user has permission to run sql queries so they are
|
||||
* given preliminary authorization. */
|
||||
audit(true, SQL_ACTION_NAME, "wrong_access", empty()),
|
||||
/* But as soon as they attempt to resolve an index that
|
||||
* they don't have access to they get denied. */
|
||||
audit(false, SQL_INDICES_ACTION_NAME, "wrong_access", hasItems("test")));
|
||||
}
|
||||
|
||||
public void testDescribeSingleFieldGranted() throws Exception {
|
||||
createUser("only_a", "read_test_a");
|
||||
|
||||
Map<String, Object> expected = new HashMap<>();
|
||||
expected.put("columns", Arrays.asList(
|
||||
columnInfo("column", "keyword"),
|
||||
columnInfo("type", "keyword")));
|
||||
expected.put("rows", singletonList(Arrays.asList("a", "BIGINT")));
|
||||
expected.put("size", 1);
|
||||
|
||||
assertResponse(expected, runSql("DESCRIBE test", "only_a"));
|
||||
assertAuditForSqlGetTableSyncGranted("only_a", "test");
|
||||
}
|
||||
|
||||
public void testDescribeSingleFieldExcepted() throws Exception {
|
||||
createUser("not_c", "read_test_a_and_b");
|
||||
|
||||
Map<String, Object> expected = new HashMap<>();
|
||||
expected.put("columns", Arrays.asList(
|
||||
columnInfo("column", "keyword"),
|
||||
columnInfo("type", "keyword")));
|
||||
expected.put("rows", Arrays.asList(
|
||||
Arrays.asList("a", "BIGINT"),
|
||||
Arrays.asList("b", "BIGINT")));
|
||||
expected.put("size", 2);
|
||||
|
||||
assertResponse(expected, runSql("DESCRIBE test", "not_c"));
|
||||
assertAuditForSqlGetTableSyncGranted("not_c", "test");
|
||||
}
|
||||
|
||||
public void testDescribeDocumentExclued() throws Exception {
|
||||
createUser("no_3s", "read_test_without_c_3");
|
||||
|
||||
assertResponse(runSql("DESCRIBE test", null), runSql("DESCRIBE test", "no_3s"));
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
assertAuditForSqlGetTableSyncGranted("no_3s", "test");
|
||||
}
|
||||
|
||||
private void expectBadRequest(ThrowingRunnable code, Matcher<String> errorMessageMatcher) {
|
||||
ResponseException e = expectThrows(ResponseException.class, code);
|
||||
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
|
||||
assertThat(e.getMessage(), errorMessageMatcher);
|
||||
}
|
||||
|
||||
private void assertResponse(Map<String, Object> expected, Map<String, Object> actual) {
|
||||
if (false == expected.equals(actual)) {
|
||||
NotEqualMessageBuilder message = new NotEqualMessageBuilder();
|
||||
message.compareMaps(actual, expected);
|
||||
fail("Response does not match:\n" + message.toString());
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> runSql(String sql, @Nullable String asUser) throws IOException {
|
||||
Header[] headers = asUser == null ? new Header[0] : new Header[] {new BasicHeader("es-security-runas-user", asUser)};
|
||||
Response response = client().performRequest("POST", "/_sql", emptyMap(),
|
||||
new StringEntity("{\"query\": \"" + sql + "\"}", ContentType.APPLICATION_JSON),
|
||||
headers);
|
||||
return toMap(response);
|
||||
}
|
||||
|
||||
private Map<String, Object> toMap(Response response) throws IOException {
|
||||
try (InputStream content = response.getEntity().getContent()) {
|
||||
return XContentHelper.convertToMap(JsonXContent.jsonXContent, content, false);
|
||||
}
|
||||
}
|
||||
|
||||
private void createUser(String name, String role) throws IOException {
|
||||
XContentBuilder user = JsonXContent.contentBuilder().prettyPrint().startObject(); {
|
||||
user.field("password", "not_used");
|
||||
user.field("roles", role);
|
||||
}
|
||||
user.endObject();
|
||||
client().performRequest("PUT", "/_xpack/security/user/" + name, emptyMap(),
|
||||
new StringEntity(user.string(), ContentType.APPLICATION_JSON));
|
||||
}
|
||||
|
||||
private void assertAuditForSqlGetTableSyncGranted(String user, String index) throws Exception {
|
||||
assertAuditEvents(
|
||||
audit(true, SQL_ACTION_NAME, user, empty()),
|
||||
audit(true, SQL_ACTION_NAME, user, hasItems(index)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts that audit events have been logged that match all the provided checkers.
|
||||
*/
|
||||
@SafeVarargs
|
||||
private final void assertAuditEvents(CheckedFunction<Map<?, ?>, Boolean, Exception>... eventCheckers) throws Exception {
|
||||
assertFalse("Previous test had an audit-related failure. All subsequent audit related assertions are bogus because we can't "
|
||||
+ "guarantee that we fully cleaned up after the last test.", auditFailure);
|
||||
try {
|
||||
assertBusy(() -> {
|
||||
SecurityManager sm = System.getSecurityManager();
|
||||
if (sm != null) {
|
||||
sm.checkPermission(new SpecialPermission());
|
||||
}
|
||||
BufferedReader logReader = AccessController.doPrivileged((PrivilegedAction<BufferedReader>) () -> {
|
||||
try {
|
||||
return Files.newBufferedReader(AUDIT_LOG_FILE, StandardCharsets.UTF_8);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
logReader.skip(auditLogWrittenBeforeTestStart);
|
||||
|
||||
List<Map<String, Object>> logs = new ArrayList<>();
|
||||
String line;
|
||||
Pattern logPattern = Pattern.compile(
|
||||
("PART PART PART origin_type=PART, origin_address=PART, "
|
||||
+ "principal=PART, (?:run_as_principal=PART, )?(?:run_by_principal=PART, )?"
|
||||
+ "action=\\[(.*?)\\], (?:indices=PART, )?request=PART")
|
||||
.replace(" ", "\\s+").replace("PART", "\\[([^\\]]*)\\]"));
|
||||
// fail(logPattern.toString());
|
||||
while ((line = logReader.readLine()) != null) {
|
||||
java.util.regex.Matcher m = logPattern.matcher(line);
|
||||
if (false == m.matches()) {
|
||||
throw new IllegalArgumentException("Unrecognized log: " + line);
|
||||
}
|
||||
int i = 1;
|
||||
Map<String, Object> log = new HashMap<>();
|
||||
/* We *could* parse the date but leaving it in the original format makes it
|
||||
* easier to find the lines in the file that this log comes from. */
|
||||
log.put("time", m.group(i++));
|
||||
log.put("origin", m.group(i++));
|
||||
String eventType = m.group(i++);
|
||||
if (false == ("access_denied".equals(eventType) || "access_granted".equals(eventType))) {
|
||||
continue;
|
||||
}
|
||||
log.put("event_type", eventType);
|
||||
log.put("origin_type", m.group(i++));
|
||||
log.put("origin_address", m.group(i++));
|
||||
log.put("principal", m.group(i++));
|
||||
log.put("run_as_principal", m.group(i++));
|
||||
log.put("run_by_principal", m.group(i++));
|
||||
String action = m.group(i++);
|
||||
if (false == (SQL_ACTION_NAME.equals(action) || SQL_INDICES_ACTION_NAME.equals(action))) {
|
||||
continue;
|
||||
}
|
||||
log.put("action", action);
|
||||
// Use a sorted list for indices for consistent error reporting
|
||||
List<String> indices = new ArrayList<>(Strings.splitStringByCommaToSet(m.group(i++)));
|
||||
Collections.sort(indices);
|
||||
log.put("indices", indices);
|
||||
log.put("request", m.group(i++));
|
||||
logs.add(log);
|
||||
}
|
||||
verifier: for (CheckedFunction<Map<?, ?>, Boolean, Exception> eventChecker : eventCheckers) {
|
||||
for (Map<String, Object> log : logs) {
|
||||
if (eventChecker.apply(log)) {
|
||||
continue verifier;
|
||||
}
|
||||
}
|
||||
StringBuilder logsMessage = new StringBuilder();
|
||||
for (Map<String, Object> log : logs) {
|
||||
logsMessage.append('\n').append(log);
|
||||
}
|
||||
fail("Didn't find an audit event we were looking for. Found:" + logsMessage);
|
||||
}
|
||||
});
|
||||
} catch (AssertionError e) {
|
||||
auditFailure = true;
|
||||
logger.warn("Failed to find an audit log. Skipping remaining tests in this class after this the missing audit"
|
||||
+ "logs could turn up later.");
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private CheckedFunction<Map<?, ?>, Boolean, Exception> audit(boolean granted, String action,
|
||||
String principal, Matcher<? extends Iterable<? extends String>> indicesMatcher) {
|
||||
String eventType = granted ? "access_granted" : "access_denied";
|
||||
return m -> eventType.equals(m.get("event_type"))
|
||||
&& action.equals(m.get("action"))
|
||||
&& principal.equals(m.get("principal"))
|
||||
&& indicesMatcher.matches(m.get("indices"));
|
||||
public RestSqlSecurityIT() {
|
||||
super(new RestActions());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,483 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
package org.elasticsearch.xpack.qa.sql.security;
|
||||
|
||||
import org.apache.http.Header;
|
||||
import org.apache.http.entity.ContentType;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.message.BasicHeader;
|
||||
import org.apache.lucene.util.SuppressForbidden;
|
||||
import org.elasticsearch.SpecialPermission;
|
||||
import org.elasticsearch.client.Response;
|
||||
import org.elasticsearch.client.ResponseException;
|
||||
import org.elasticsearch.common.CheckedFunction;
|
||||
import org.elasticsearch.common.Nullable;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentHelper;
|
||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||
import org.elasticsearch.test.NotEqualMessageBuilder;
|
||||
import org.elasticsearch.test.rest.ESRestTestCase;
|
||||
import org.hamcrest.Matcher;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.security.AccessController;
|
||||
import java.security.PrivilegedAction;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static java.util.Collections.emptyList;
|
||||
import static java.util.Collections.emptyMap;
|
||||
import static java.util.Collections.singletonList;
|
||||
import static java.util.Collections.singletonMap;
|
||||
import static org.elasticsearch.xpack.qa.sql.rest.RestSqlTestCase.columnInfo;
|
||||
import static org.hamcrest.Matchers.contains;
|
||||
import static org.hamcrest.Matchers.empty;
|
||||
import static org.hamcrest.Matchers.hasItems;
|
||||
|
||||
public abstract class SqlSecurityTestCase extends ESRestTestCase {
|
||||
/**
|
||||
* Actions taken by this test.
|
||||
*/
|
||||
protected interface Actions {
|
||||
void queryWorksAsAdmin() throws Exception;
|
||||
void expectMatchesAdmin(String adminSql, String user, String userSql) throws Exception;
|
||||
void expectDescribe(Map<String, String> columns, String user) throws Exception;
|
||||
void expectShowTables(List<String> tables, String user) throws Exception;
|
||||
|
||||
void expectForbidden(String user, String sql) throws Exception;
|
||||
void expectUnknownColumn(String user, String sql, String column) throws Exception;
|
||||
}
|
||||
|
||||
private static final String SQL_ACTION_NAME = "indices:data/read/sql";
|
||||
private static final String SQL_INDICES_ACTION_NAME = "indices:data/read/sql/tables";
|
||||
/**
|
||||
* Location of the audit log file. We could technically figure this out by reading the admin
|
||||
* APIs but it isn't worth doing because we also have to give ourselves permission to read
|
||||
* the file and that must be done by setting a system property and reading it in
|
||||
* {@code plugin-security.policy}. So we may as well have gradle set the property.
|
||||
*/
|
||||
private static final Path AUDIT_LOG_FILE = lookupAuditLog();
|
||||
|
||||
@SuppressForbidden(reason="security doesn't work with mock filesystem")
|
||||
private static Path lookupAuditLog() {
|
||||
String auditLogFileString = System.getProperty("tests.audit.logfile");
|
||||
if (null == auditLogFileString) {
|
||||
throw new IllegalStateException("tests.audit.logfile must be set to run this test. It is automatically "
|
||||
+ "set by gradle. If you must set it yourself then it should be the absolute path to the audit "
|
||||
+ "log file generated by running x-pack with audit logging enabled.");
|
||||
}
|
||||
return Paths.get(auditLogFileString);
|
||||
}
|
||||
|
||||
private static boolean oneTimeSetup = false;
|
||||
private static boolean auditFailure = false;
|
||||
|
||||
/**
|
||||
* The actions taken by this test.
|
||||
*/
|
||||
private final Actions actions;
|
||||
|
||||
/**
|
||||
* How much of the audit log was written before the test started.
|
||||
*/
|
||||
private long auditLogWrittenBeforeTestStart;
|
||||
|
||||
public SqlSecurityTestCase(Actions actions) {
|
||||
this.actions = actions;
|
||||
}
|
||||
|
||||
/**
|
||||
* All tests run as a an administrative user but use
|
||||
* <code>es-security-runas-user</code> to become a less privileged user when needed.
|
||||
*/
|
||||
@Override
|
||||
protected Settings restClientSettings() {
|
||||
return RestSqlIT.securitySettings();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean preserveIndicesUponCompletion() {
|
||||
/* We can't wipe the cluster between tests because that nukes the audit
|
||||
* trail index which makes the auditing flaky. Instead we wipe all
|
||||
* indices after the entire class is finished. */
|
||||
return true;
|
||||
}
|
||||
|
||||
@Before
|
||||
public void oneTimeSetup() throws Exception {
|
||||
if (oneTimeSetup) {
|
||||
/* Since we don't wipe the cluster between tests we only need to
|
||||
* write the test data once. */
|
||||
return;
|
||||
}
|
||||
StringBuilder bulk = new StringBuilder();
|
||||
bulk.append("{\"index\":{\"_index\": \"test\", \"_type\": \"doc\", \"_id\":\"1\"}\n");
|
||||
bulk.append("{\"a\": 1, \"b\": 2, \"c\": 3}\n");
|
||||
bulk.append("{\"index\":{\"_index\": \"test\", \"_type\": \"doc\", \"_id\":\"2\"}\n");
|
||||
bulk.append("{\"a\": 4, \"b\": 5, \"c\": 6}\n");
|
||||
bulk.append("{\"index\":{\"_index\": \"bort\", \"_type\": \"doc\", \"_id\":\"1\"}\n");
|
||||
bulk.append("{\"a\": \"test\"}\n");
|
||||
client().performRequest("PUT", "/_bulk", singletonMap("refresh", "true"),
|
||||
new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON));
|
||||
oneTimeSetup = true;
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setInitialAuditLogOffset() throws IOException {
|
||||
SecurityManager sm = System.getSecurityManager();
|
||||
if (sm != null) {
|
||||
sm.checkPermission(new SpecialPermission());
|
||||
}
|
||||
AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
|
||||
if (false == Files.exists(AUDIT_LOG_FILE)) {
|
||||
auditLogWrittenBeforeTestStart = 0;
|
||||
return null;
|
||||
}
|
||||
if (false == Files.isRegularFile(AUDIT_LOG_FILE)) {
|
||||
throw new IllegalStateException("expected tests.audit.logfile [" + AUDIT_LOG_FILE + "]to be a plain file but wasn't");
|
||||
}
|
||||
try {
|
||||
auditLogWrittenBeforeTestStart = Files.size(AUDIT_LOG_FILE);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void wipeIndicesAfterTests() throws IOException {
|
||||
try {
|
||||
adminClient().performRequest("DELETE", "*");
|
||||
} catch (ResponseException e) {
|
||||
// 404 here just means we had no indexes
|
||||
if (e.getResponse().getStatusLine().getStatusCode() != 404) {
|
||||
throw e;
|
||||
}
|
||||
} finally {
|
||||
// Clear the static state so other subclasses can reuse it later
|
||||
oneTimeSetup = false;
|
||||
auditFailure = false;
|
||||
}
|
||||
}
|
||||
|
||||
// NOCOMMIT we'll have to test scrolling as well
|
||||
// NOCOMMIT assert that we don't have more audit logs then what we expect.
|
||||
|
||||
public void testQueryWorksAsAdmin() throws Exception {
|
||||
actions.queryWorksAsAdmin();
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
}
|
||||
|
||||
public void testQueryWithFullAccess() throws Exception {
|
||||
createUser("full_access", "read_all");
|
||||
|
||||
actions.expectMatchesAdmin("SELECT * FROM test ORDER BY a", "full_access", "SELECT * FROM test ORDER BY a");
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
assertAuditForSqlGetTableSyncGranted("full_access", "test");
|
||||
}
|
||||
|
||||
public void testQueryNoAccess() throws Exception {
|
||||
createUser("no_access", "read_nothing");
|
||||
|
||||
actions.expectForbidden("no_access", "SELECT * FROM test");
|
||||
assertAuditEvents(audit(false, SQL_ACTION_NAME, "no_access", empty()));
|
||||
}
|
||||
|
||||
public void testQueryWrongAccess() throws Exception {
|
||||
createUser("wrong_access", "read_something_else");
|
||||
|
||||
actions.expectForbidden("wrong_access", "SELECT * FROM test");
|
||||
assertAuditEvents(
|
||||
/* This user has permission to run sql queries so they are
|
||||
* given preliminary authorization. */
|
||||
audit(true, SQL_ACTION_NAME, "wrong_access", empty()),
|
||||
/* But as soon as they attempt to resolve an index that
|
||||
* they don't have access to they get denied. */
|
||||
audit(false, SQL_ACTION_NAME, "wrong_access", hasItems("test")));
|
||||
}
|
||||
|
||||
public void testQuerySingleFieldGranted() throws Exception {
|
||||
createUser("only_a", "read_test_a");
|
||||
|
||||
actions.expectMatchesAdmin("SELECT a FROM test", "only_a", "SELECT * FROM test");
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
assertAuditForSqlGetTableSyncGranted("only_a", "test");
|
||||
}
|
||||
|
||||
public void testQueryStringSingeFieldGrantedWrongRequested() throws Exception {
|
||||
createUser("only_a", "read_test_a");
|
||||
|
||||
actions.expectUnknownColumn("only_a", "SELECT c FROM test", "c");
|
||||
/* The user has permission to query the index but one of the
|
||||
* columns that they explicitly mention is hidden from them
|
||||
* by field level access control. This *looks* like a successful
|
||||
* query from the audit side because all the permissions checked
|
||||
* out but it failed in SQL because it couldn't compile the
|
||||
* query without the metadata for the missing field. */
|
||||
assertAuditForSqlGetTableSyncGranted("only_a", "test");
|
||||
}
|
||||
|
||||
public void testQuerySingleFieldExcepted() throws Exception {
|
||||
createUser("not_c", "read_test_a_and_b");
|
||||
|
||||
actions.expectMatchesAdmin("SELECT a, b FROM test", "not_c", "SELECT * FROM test");
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
assertAuditForSqlGetTableSyncGranted("not_c", "test");
|
||||
}
|
||||
|
||||
public void testQuerySingleFieldExceptionedWrongRequested() throws Exception {
|
||||
createUser("not_c", "read_test_a_and_b");
|
||||
|
||||
actions.expectUnknownColumn("not_c", "SELECT c FROM test", "c");
|
||||
/* The user has permission to query the index but one of the
|
||||
* columns that they explicitly mention is hidden from them
|
||||
* by field level access control. This *looks* like a successful
|
||||
* query from the audit side because all the permissions checked
|
||||
* out but it failed in SQL because it couldn't compile the
|
||||
* query without the metadata for the missing field. */
|
||||
assertAuditForSqlGetTableSyncGranted("not_c", "test");
|
||||
}
|
||||
|
||||
public void testQueryDocumentExclued() throws Exception {
|
||||
createUser("no_3s", "read_test_without_c_3");
|
||||
|
||||
actions.expectMatchesAdmin("SELECT * FROM test WHERE c != 3", "no_3s", "SELECT * FROM test");
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
assertAuditForSqlGetTableSyncGranted("no_3s", "test");
|
||||
}
|
||||
|
||||
public void testShowTablesWorksAsAdmin() throws Exception {
|
||||
actions.expectShowTables(Arrays.asList("bort", "test"), null);
|
||||
assertAuditEvents(
|
||||
audit(true, SQL_ACTION_NAME, "test_admin", empty()),
|
||||
audit(true, SQL_INDICES_ACTION_NAME, "test_admin", hasItems("test", "bort")));
|
||||
}
|
||||
|
||||
public void testShowTablesWorksAsFullAccess() throws Exception {
|
||||
createUser("full_access", "read_all");
|
||||
|
||||
actions.expectMatchesAdmin("SHOW TABLES", "full_access", "SHOW TABLES");
|
||||
assertAuditEvents(
|
||||
audit(true, SQL_ACTION_NAME, "test_admin", empty()),
|
||||
audit(true, SQL_INDICES_ACTION_NAME, "test_admin", hasItems("test", "bort")),
|
||||
audit(true, SQL_ACTION_NAME, "full_access", empty()),
|
||||
audit(true, SQL_INDICES_ACTION_NAME, "full_access", hasItems("test", "bort")));
|
||||
}
|
||||
|
||||
public void testShowTablesWithNoAccess() throws Exception {
|
||||
createUser("no_access", "read_nothing");
|
||||
|
||||
actions.expectForbidden("no_access", "SHOW TABLES");
|
||||
assertAuditEvents(audit(false, SQL_ACTION_NAME, "no_access", empty()));
|
||||
}
|
||||
|
||||
public void testShowTablesWithLimitedAccess() throws Exception {
|
||||
createUser("read_bort", "read_bort");
|
||||
|
||||
actions.expectMatchesAdmin("SHOW TABLES LIKE 'bort'", "read_bort", "SHOW TABLES");
|
||||
assertAuditEvents(
|
||||
audit(true, SQL_ACTION_NAME, "test_admin", empty()),
|
||||
audit(true, SQL_INDICES_ACTION_NAME, "test_admin", contains("bort")),
|
||||
audit(true, SQL_ACTION_NAME, "read_bort", empty()),
|
||||
audit(true, SQL_INDICES_ACTION_NAME, "read_bort", contains("bort")));
|
||||
}
|
||||
|
||||
public void testShowTablesWithLimitedAccessAndPattern() throws Exception {
|
||||
createUser("read_bort", "read_bort");
|
||||
|
||||
actions.expectMatchesAdmin("SHOW TABLES LIKE 'not_created'", "read_bort", "SHOW TABLES LIKE 'test'");
|
||||
assertAuditEvents(
|
||||
audit(true, SQL_ACTION_NAME, "read_bort", empty()),
|
||||
audit(true, SQL_INDICES_ACTION_NAME, "read_bort", contains("*", "-*")));
|
||||
}
|
||||
|
||||
public void testDescribeWorksAsAdmin() throws Exception {
|
||||
Map<String, String> expected = new TreeMap<>();
|
||||
expected.put("a", "BIGINT");
|
||||
expected.put("b", "BIGINT");
|
||||
expected.put("c", "BIGINT");
|
||||
actions.expectDescribe(expected, null);
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
}
|
||||
|
||||
public void testDescribeWorksAsFullAccess() throws Exception {
|
||||
createUser("full_access", "read_all");
|
||||
|
||||
actions.expectMatchesAdmin("DESCRIBE test", "full_access", "DESCRIBE test");
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
assertAuditForSqlGetTableSyncGranted("full_access", "test");
|
||||
}
|
||||
|
||||
public void testDescribeWithNoAccess() throws Exception {
|
||||
createUser("no_access", "read_nothing");
|
||||
|
||||
actions.expectForbidden("no_access", "DESCRIBE test");
|
||||
assertAuditEvents(audit(false, SQL_ACTION_NAME, "no_access", empty()));
|
||||
}
|
||||
|
||||
public void testDescribeWithWrongAccess() throws Exception {
|
||||
createUser("wrong_access", "read_something_else");
|
||||
|
||||
actions.expectForbidden("wrong_access", "DESCRIBE test");
|
||||
assertAuditEvents(
|
||||
/* This user has permission to run sql queries so they are
|
||||
* given preliminary authorization. */
|
||||
audit(true, SQL_ACTION_NAME, "wrong_access", empty()),
|
||||
/* But as soon as they attempt to resolve an index that
|
||||
* they don't have access to they get denied. */
|
||||
audit(false, SQL_INDICES_ACTION_NAME, "wrong_access", hasItems("test")));
|
||||
}
|
||||
|
||||
public void testDescribeSingleFieldGranted() throws Exception {
|
||||
createUser("only_a", "read_test_a");
|
||||
|
||||
actions.expectDescribe(singletonMap("a", "BIGINT"), "only_a");
|
||||
assertAuditForSqlGetTableSyncGranted("only_a", "test");
|
||||
}
|
||||
|
||||
public void testDescribeSingleFieldExcepted() throws Exception {
|
||||
createUser("not_c", "read_test_a_and_b");
|
||||
|
||||
Map<String, String> expected = new TreeMap<>();
|
||||
expected.put("a", "BIGINT");
|
||||
expected.put("b", "BIGINT");
|
||||
actions.expectDescribe(expected, "not_c");
|
||||
assertAuditForSqlGetTableSyncGranted("not_c", "test");
|
||||
}
|
||||
|
||||
public void testDescribeDocumentExclued() throws Exception {
|
||||
createUser("no_3s", "read_test_without_c_3");
|
||||
|
||||
actions.expectMatchesAdmin("DESCRIBE test", "no_3s", "DESCRIBE test");
|
||||
assertAuditForSqlGetTableSyncGranted("test_admin", "test");
|
||||
assertAuditForSqlGetTableSyncGranted("no_3s", "test");
|
||||
}
|
||||
|
||||
private void createUser(String name, String role) throws IOException {
|
||||
XContentBuilder user = JsonXContent.contentBuilder().prettyPrint().startObject(); {
|
||||
user.field("password", "testpass");
|
||||
user.field("roles", role);
|
||||
}
|
||||
user.endObject();
|
||||
client().performRequest("PUT", "/_xpack/security/user/" + name, emptyMap(),
|
||||
new StringEntity(user.string(), ContentType.APPLICATION_JSON));
|
||||
}
|
||||
|
||||
private void assertAuditForSqlGetTableSyncGranted(String user, String index) throws Exception {
|
||||
assertAuditEvents(
|
||||
audit(true, SQL_ACTION_NAME, user, empty()),
|
||||
audit(true, SQL_ACTION_NAME, user, hasItems(index)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts that audit events have been logged that match all the provided checkers.
|
||||
*/
|
||||
@SafeVarargs
|
||||
private final void assertAuditEvents(CheckedFunction<Map<?, ?>, Boolean, Exception>... eventCheckers) throws Exception {
|
||||
assertFalse("Previous test had an audit-related failure. All subsequent audit related assertions are bogus because we can't "
|
||||
+ "guarantee that we fully cleaned up after the last test.", auditFailure);
|
||||
try {
|
||||
assertBusy(() -> {
|
||||
SecurityManager sm = System.getSecurityManager();
|
||||
if (sm != null) {
|
||||
sm.checkPermission(new SpecialPermission());
|
||||
}
|
||||
BufferedReader logReader = AccessController.doPrivileged((PrivilegedAction<BufferedReader>) () -> {
|
||||
try {
|
||||
return Files.newBufferedReader(AUDIT_LOG_FILE, StandardCharsets.UTF_8);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
logReader.skip(auditLogWrittenBeforeTestStart);
|
||||
|
||||
List<Map<String, Object>> logs = new ArrayList<>();
|
||||
String line;
|
||||
Pattern logPattern = Pattern.compile(
|
||||
("PART PART PART origin_type=PART, origin_address=PART, "
|
||||
+ "principal=PART, (?:run_as_principal=PART, )?(?:run_by_principal=PART, )?"
|
||||
+ "action=\\[(.*?)\\], (?:indices=PART, )?request=PART")
|
||||
.replace(" ", "\\s+").replace("PART", "\\[([^\\]]*)\\]"));
|
||||
// fail(logPattern.toString());
|
||||
while ((line = logReader.readLine()) != null) {
|
||||
java.util.regex.Matcher m = logPattern.matcher(line);
|
||||
if (false == m.matches()) {
|
||||
throw new IllegalArgumentException("Unrecognized log: " + line);
|
||||
}
|
||||
int i = 1;
|
||||
Map<String, Object> log = new HashMap<>();
|
||||
/* We *could* parse the date but leaving it in the original format makes it
|
||||
* easier to find the lines in the file that this log comes from. */
|
||||
log.put("time", m.group(i++));
|
||||
log.put("origin", m.group(i++));
|
||||
String eventType = m.group(i++);
|
||||
if (false == ("access_denied".equals(eventType) || "access_granted".equals(eventType))) {
|
||||
continue;
|
||||
}
|
||||
log.put("event_type", eventType);
|
||||
log.put("origin_type", m.group(i++));
|
||||
log.put("origin_address", m.group(i++));
|
||||
log.put("principal", m.group(i++));
|
||||
log.put("run_as_principal", m.group(i++));
|
||||
log.put("run_by_principal", m.group(i++));
|
||||
String action = m.group(i++);
|
||||
if (false == (SQL_ACTION_NAME.equals(action) || SQL_INDICES_ACTION_NAME.equals(action))) {
|
||||
continue;
|
||||
}
|
||||
log.put("action", action);
|
||||
// Use a sorted list for indices for consistent error reporting
|
||||
List<String> indices = new ArrayList<>(Strings.splitStringByCommaToSet(m.group(i++)));
|
||||
Collections.sort(indices);
|
||||
log.put("indices", indices);
|
||||
log.put("request", m.group(i++));
|
||||
logs.add(log);
|
||||
}
|
||||
verifier: for (CheckedFunction<Map<?, ?>, Boolean, Exception> eventChecker : eventCheckers) {
|
||||
for (Map<String, Object> log : logs) {
|
||||
if (eventChecker.apply(log)) {
|
||||
continue verifier;
|
||||
}
|
||||
}
|
||||
StringBuilder logsMessage = new StringBuilder();
|
||||
for (Map<String, Object> log : logs) {
|
||||
logsMessage.append('\n').append(log);
|
||||
}
|
||||
fail("Didn't find an audit event we were looking for. Found:" + logsMessage);
|
||||
}
|
||||
});
|
||||
} catch (AssertionError e) {
|
||||
auditFailure = true;
|
||||
logger.warn("Failed to find an audit log. Skipping remaining tests in this class after this the missing audit"
|
||||
+ "logs could turn up later.");
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private CheckedFunction<Map<?, ?>, Boolean, Exception> audit(boolean granted, String action,
|
||||
String principal, Matcher<? extends Iterable<? extends String>> indicesMatcher) {
|
||||
String eventType = granted ? "access_granted" : "access_denied";
|
||||
return m -> eventType.equals(m.get("event_type"))
|
||||
&& action.equals(m.get("action"))
|
||||
&& principal.equals(m.get("principal"))
|
||||
&& indicesMatcher.matches(m.get("indices"));
|
||||
}
|
||||
}
|
||||
|
|
@ -8,7 +8,6 @@ package org.elasticsearch.xpack.qa.sql.cli;
|
|||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.entity.ContentType;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.elasticsearch.SpecialPermission;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.common.Booleans;
|
||||
import org.elasticsearch.common.CheckedConsumer;
|
||||
|
|
@ -24,48 +23,14 @@ import org.junit.Before;
|
|||
import org.junit.ClassRule;
|
||||
import org.junit.rules.ExternalResource;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.PrintWriter;
|
||||
import java.net.InetAddress;
|
||||
import java.net.Socket;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.AccessControlException;
|
||||
import java.security.AccessController;
|
||||
import java.security.PrivilegedAction;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import static java.util.Collections.singletonMap;
|
||||
import static org.hamcrest.Matchers.empty;
|
||||
import static org.hamcrest.Matchers.endsWith;
|
||||
import static org.hamcrest.Matchers.not;
|
||||
import static org.hamcrest.Matchers.startsWith;
|
||||
|
||||
public abstract class CliIntegrationTestCase extends ESRestTestCase {
|
||||
private static final InetAddress CLI_FIXTURE_ADDRESS;
|
||||
private static final int CLI_FIXTURE_PORT;
|
||||
static {
|
||||
String addressAndPort = System.getProperty("tests.cli.fixture");
|
||||
if (addressAndPort == null) {
|
||||
throw new IllegalArgumentException("Must set the [tests.cli.fixture] property. Gradle handles this for you "
|
||||
+ " in regular tests. In embedded mode the easiest thing to do is run "
|
||||
+ "`gradle :x-pack-elasticsearch:qa:sql:no-security:run` and to set the property to the contents of "
|
||||
+ "`qa/sql/no-security/build/fixtures/cliFixture/ports`");
|
||||
}
|
||||
int split = addressAndPort.lastIndexOf(':');
|
||||
try {
|
||||
CLI_FIXTURE_ADDRESS = InetAddress.getByName(addressAndPort.substring(0, split));
|
||||
} catch (UnknownHostException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
CLI_FIXTURE_PORT = Integer.parseInt(addressAndPort.substring(split + 1));
|
||||
}
|
||||
|
||||
/**
|
||||
* Should the HTTP server that serves SQL be embedded in the test
|
||||
* process (true) or should the JDBC driver connect to Elasticsearch
|
||||
|
|
@ -82,68 +47,32 @@ public abstract class CliIntegrationTestCase extends ESRestTestCase {
|
|||
public static final EmbeddedCliServer EMBEDDED = EMBED_SQL ? new EmbeddedCliServer() : null;
|
||||
public static final Supplier<String> ES = EMBED_SQL ? EMBEDDED::address : CliIntegrationTestCase::elasticsearchAddress;
|
||||
|
||||
private Socket cliSocket;
|
||||
private PrintWriter out;
|
||||
private BufferedReader in;
|
||||
/**
|
||||
* Read an address for Elasticsearch suitable for the CLI from the system properties.
|
||||
*/
|
||||
public static String elasticsearchAddress() {
|
||||
String cluster = System.getProperty("tests.rest.cluster");
|
||||
// CLI only supports a single node at a time so we just give it one.
|
||||
return cluster.split(",")[0];
|
||||
}
|
||||
|
||||
private RemoteCli cli;
|
||||
|
||||
/**
|
||||
* Asks the CLI Fixture to start a CLI instance.
|
||||
*/
|
||||
@Before
|
||||
public void startCli() throws IOException {
|
||||
SecurityManager sm = System.getSecurityManager();
|
||||
if (sm != null) {
|
||||
sm.checkPermission(new SpecialPermission());
|
||||
}
|
||||
logger.info("connecting to the cli fixture at {}:{}", CLI_FIXTURE_ADDRESS, CLI_FIXTURE_PORT);
|
||||
cliSocket = AccessController.doPrivileged(new PrivilegedAction<Socket>() {
|
||||
@Override
|
||||
public Socket run() {
|
||||
try {
|
||||
return new Socket(CLI_FIXTURE_ADDRESS, CLI_FIXTURE_PORT);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
});
|
||||
logger.info("connected");
|
||||
cliSocket.setSoTimeout(10000);
|
||||
|
||||
out = new PrintWriter(new OutputStreamWriter(cliSocket.getOutputStream(), StandardCharsets.UTF_8), true);
|
||||
out.println(esUrlPrefix() + ES.get());
|
||||
in = new BufferedReader(new InputStreamReader(cliSocket.getInputStream(), StandardCharsets.UTF_8));
|
||||
// Throw out the logo and warnings about making a dumb terminal
|
||||
while (false == readLine().contains("SQL"));
|
||||
// Throw out the empty line before all the good stuff
|
||||
assertEquals("", readLine());
|
||||
cli = new RemoteCli(esUrlPrefix() + ES.get());
|
||||
}
|
||||
|
||||
@After
|
||||
public void orderlyShutdown() throws IOException, InterruptedException {
|
||||
if (cliSocket == null) {
|
||||
if (cli == null) {
|
||||
// failed to connect to the cli so there is nothing to do here
|
||||
return;
|
||||
}
|
||||
try {
|
||||
// Try and shutdown the client normally
|
||||
/* Don't use println because it enits \r\n on windows but we put the
|
||||
* terminal in unix mode to make the tests consistent. */
|
||||
out.print("quit;\n");
|
||||
out.flush();
|
||||
List<String> nonQuit = new ArrayList<>();
|
||||
String line;
|
||||
while (false == (line = readLine()).startsWith("[?1h=[33msql> [0mquit;[90mBye![0m")) {
|
||||
if (false == line.isEmpty()) {
|
||||
nonQuit.add(line);
|
||||
}
|
||||
}
|
||||
assertThat("unconsumed lines", nonQuit, empty());
|
||||
} finally {
|
||||
out.close();
|
||||
in.close();
|
||||
// Most importantly, close the socket so the next test can use the fixture
|
||||
cliSocket.close();
|
||||
}
|
||||
cli.close();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -162,29 +91,12 @@ public abstract class CliIntegrationTestCase extends ESRestTestCase {
|
|||
client().performRequest("PUT", "/" + index + "/doc/1", singletonMap("refresh", "true"), doc);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a command and assert the echo.
|
||||
*/
|
||||
protected String command(String command) throws IOException {
|
||||
assertThat("; automatically added", command, not(endsWith(";")));
|
||||
logger.info("out: {};", command);
|
||||
/* Don't use println because it enits \r\n on windows but we put the
|
||||
* terminal in unix mode to make the tests consistent. */
|
||||
out.print(command + ";\n");
|
||||
out.flush();
|
||||
String firstResponse = "[?1h=[33msql> [0m" + command + ";";
|
||||
String firstLine = readLine();
|
||||
assertThat(firstLine, startsWith(firstResponse));
|
||||
return firstLine.substring(firstResponse.length());
|
||||
public String command(String command) throws IOException {
|
||||
return cli.command(command);
|
||||
}
|
||||
|
||||
protected String readLine() throws IOException {
|
||||
/* Since we can't *see* esc in the error messages we just
|
||||
* remove it here and pretend it isn't required. Hopefully
|
||||
* `[` is enough for us to assert on. */
|
||||
String line = in.readLine().replace("\u001B", "");
|
||||
logger.info("in : {}", line);
|
||||
return line;
|
||||
public String readLine() throws IOException {
|
||||
return cli.readLine();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -228,10 +140,4 @@ public abstract class CliIntegrationTestCase extends ESRestTestCase {
|
|||
return server.address().getAddress().getHostAddress() + ":" + server.address().getPort();
|
||||
}
|
||||
}
|
||||
|
||||
private static String elasticsearchAddress() {
|
||||
String cluster = System.getProperty("tests.rest.cluster");
|
||||
// CLI only supports a single node at a time so we just give it one.
|
||||
return cluster.split(",")[0];
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,139 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
package org.elasticsearch.xpack.qa.sql.cli;
|
||||
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.SpecialPermission;
|
||||
import org.elasticsearch.common.logging.Loggers;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.PrintWriter;
|
||||
import java.net.InetAddress;
|
||||
import java.net.Socket;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.AccessController;
|
||||
import java.security.PrivilegedAction;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.hamcrest.Matchers.empty;
|
||||
import static org.hamcrest.Matchers.endsWith;
|
||||
import static org.hamcrest.Matchers.not;
|
||||
import static org.hamcrest.Matchers.startsWith;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertThat;
|
||||
|
||||
public class RemoteCli implements Closeable {
|
||||
private static final Logger logger = Loggers.getLogger(RemoteCli.class);
|
||||
|
||||
private static final InetAddress CLI_FIXTURE_ADDRESS;
|
||||
private static final int CLI_FIXTURE_PORT;
|
||||
static {
|
||||
String addressAndPort = System.getProperty("tests.cli.fixture");
|
||||
if (addressAndPort == null) {
|
||||
throw new IllegalArgumentException("Must set the [tests.cli.fixture] property. Gradle handles this for you "
|
||||
+ " in regular tests. In embedded mode the easiest thing to do is run "
|
||||
+ "`gradle :x-pack-elasticsearch:qa:sql:no-security:run` and to set the property to the contents of "
|
||||
+ "`qa/sql/no-security/build/fixtures/cliFixture/ports`");
|
||||
}
|
||||
int split = addressAndPort.lastIndexOf(':');
|
||||
try {
|
||||
CLI_FIXTURE_ADDRESS = InetAddress.getByName(addressAndPort.substring(0, split));
|
||||
} catch (UnknownHostException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
CLI_FIXTURE_PORT = Integer.parseInt(addressAndPort.substring(split + 1));
|
||||
}
|
||||
|
||||
private final Socket socket;
|
||||
private final PrintWriter out;
|
||||
private final BufferedReader in;
|
||||
|
||||
public RemoteCli(String elasticsearchAddress) throws IOException {
|
||||
SecurityManager sm = System.getSecurityManager();
|
||||
if (sm != null) {
|
||||
sm.checkPermission(new SpecialPermission());
|
||||
}
|
||||
logger.info("connecting to the cli fixture at {}:{}", CLI_FIXTURE_ADDRESS, CLI_FIXTURE_PORT);
|
||||
socket = AccessController.doPrivileged(new PrivilegedAction<Socket>() {
|
||||
@Override
|
||||
public Socket run() {
|
||||
try {
|
||||
return new Socket(CLI_FIXTURE_ADDRESS, CLI_FIXTURE_PORT);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
});
|
||||
logger.info("connected");
|
||||
socket.setSoTimeout(10000);
|
||||
|
||||
out = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8), true);
|
||||
out.println(elasticsearchAddress);
|
||||
in = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8));
|
||||
// Throw out the logo and warnings about making a dumb terminal
|
||||
while (false == readLine().contains("SQL"));
|
||||
// Throw out the empty line before all the good stuff
|
||||
assertEquals("", readLine());
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts an orderly shutdown of the CLI, reporting any unconsumed lines as errors.
|
||||
*/
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
try {
|
||||
// Try and shutdown the client normally
|
||||
/* Don't use println because it enits \r\n on windows but we put the
|
||||
* terminal in unix mode to make the tests consistent. */
|
||||
out.print("quit;\n");
|
||||
out.flush();
|
||||
List<String> nonQuit = new ArrayList<>();
|
||||
String line;
|
||||
while (false == (line = readLine()).startsWith("[?1h=[33msql> [0mquit;[90mBye![0m")) {
|
||||
if (false == line.isEmpty()) {
|
||||
nonQuit.add(line);
|
||||
}
|
||||
}
|
||||
assertThat("unconsumed lines", nonQuit, empty());
|
||||
} finally {
|
||||
out.close();
|
||||
in.close();
|
||||
// Most importantly, close the socket so the next test can use the fixture
|
||||
socket.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a command and assert the echo.
|
||||
*/
|
||||
public String command(String command) throws IOException {
|
||||
assertThat("; automatically added", command, not(endsWith(";")));
|
||||
logger.info("out: {};", command);
|
||||
/* Don't use println because it enits \r\n on windows but we put the
|
||||
* terminal in unix mode to make the tests consistent. */
|
||||
out.print(command + ";\n");
|
||||
out.flush();
|
||||
String firstResponse = "[?1h=[33msql> [0m" + command + ";";
|
||||
String firstLine = readLine();
|
||||
assertThat(firstLine, startsWith(firstResponse));
|
||||
return firstLine.substring(firstResponse.length());
|
||||
}
|
||||
|
||||
public String readLine() throws IOException {
|
||||
/* Since we can't *see* esc in the error messages we just
|
||||
* remove it here and pretend it isn't required. Hopefully
|
||||
* `[` is enough for us to assert on. */
|
||||
String line = in.readLine().replace("\u001B", "");
|
||||
logger.info("in : {}", line);
|
||||
return line;
|
||||
}
|
||||
}
|
||||
|
|
@ -42,13 +42,20 @@ public abstract class JdbcIntegrationTestCase extends ESRestTestCase {
|
|||
@ClassRule
|
||||
public static final EmbeddedJdbcServer EMBEDDED_SERVER = EMBED_SQL ? new EmbeddedJdbcServer() : null;
|
||||
|
||||
/**
|
||||
* Read an address for Elasticsearch suitable for the JDBC driver from the system properties.
|
||||
*/
|
||||
public static String elasticsearchAddress() {
|
||||
String cluster = System.getProperty("tests.rest.cluster");
|
||||
// JDBC only supports a single node at a time so we just give it one.
|
||||
return "jdbc:es://" + cluster.split(",")[0];
|
||||
}
|
||||
|
||||
public Connection esJdbc() throws SQLException {
|
||||
if (EMBED_SQL) {
|
||||
return EMBEDDED_SERVER.connection();
|
||||
}
|
||||
String cluster = System.getProperty("tests.rest.cluster");
|
||||
// We only support a single node at this time.
|
||||
return DriverManager.getConnection("jdbc:es://" + cluster.split(",")[0], connectionProperties());
|
||||
return DriverManager.getConnection(elasticsearchAddress(), connectionProperties());
|
||||
}
|
||||
|
||||
public static void index(String index, CheckedConsumer<XContentBuilder, IOException> body) throws IOException {
|
||||
|
|
|
|||
Loading…
Reference in New Issue