diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml index c762a601651..62aac48dc99 100644 --- a/.github/workflows/revised-its.yml +++ b/.github/workflows/revised-its.yml @@ -50,7 +50,7 @@ jobs: matrix: #jdk: [8, 11, 17] jdk: [8] - it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat] + it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat, Security] #indexer: [indexer, middleManager] indexer: [middleManager] uses: ./.github/workflows/reusable-revised-its.yml diff --git a/integration-tests-ex/cases/pom.xml b/integration-tests-ex/cases/pom.xml index dc871bec097..eb4822c425d 100644 --- a/integration-tests-ex/cases/pom.xml +++ b/integration-tests-ex/cases/pom.xml @@ -530,5 +530,14 @@ + + IT-Security + + false + + + Security + + diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/auth/ITSecurityBasicQuery.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/auth/ITSecurityBasicQuery.java index 9d76af07de8..181e8e92d1f 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/auth/ITSecurityBasicQuery.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/auth/ITSecurityBasicQuery.java @@ -22,8 +22,10 @@ package org.apache.druid.testsEx.auth; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; +import org.apache.druid.common.utils.IdUtils; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.http.client.response.StatusResponseHolder; +import org.apache.druid.msq.indexing.MSQControllerTask; import org.apache.druid.server.security.Action; import org.apache.druid.server.security.Resource; import org.apache.druid.server.security.ResourceAction; @@ -31,9 +33,11 @@ import org.apache.druid.sql.http.SqlQuery; import org.apache.druid.storage.local.LocalFileExportStorageProvider; import org.apache.druid.storage.s3.output.S3ExportStorageProvider; import org.apache.druid.testing.clients.CoordinatorResourceTestClient; +import org.apache.druid.testing.clients.OverlordResourceTestClient; import org.apache.druid.testing.clients.SecurityClient; import org.apache.druid.testing.utils.DataLoaderHelper; import org.apache.druid.testing.utils.MsqTestQueryHelper; +import org.apache.druid.tests.indexer.AbstractIndexerTest; import org.apache.druid.testsEx.categories.Security; import org.apache.druid.testsEx.config.DruidTestRunner; import org.jboss.netty.handler.codec.http.HttpResponseStatus; @@ -62,10 +66,13 @@ public class ITSecurityBasicQuery private CoordinatorResourceTestClient coordinatorClient; @Inject private SecurityClient securityClient; + @Inject + private OverlordResourceTestClient overlordResourceTestClient; public static final String USER_1 = "user1"; public static final String ROLE_1 = "role1"; public static final String USER_1_PASSWORD = "password1"; + private static final String EXPORT_TASK = "/indexer/export_task.json"; @Before public void setUp() throws IOException @@ -154,6 +161,9 @@ public class ITSecurityBasicQuery ); securityClient.setPermissionsToRole(ROLE_1, permissions); + // Wait for a second so that the auth is synced, to avoid flakiness + Thread.sleep(1000); + String queryLocal = StringUtils.format( "INSERT INTO %s\n" @@ -216,6 +226,9 @@ public class ITSecurityBasicQuery ); securityClient.setPermissionsToRole(ROLE_1, permissions); + // Wait for a second so that the auth is synced, to avoid flakiness + Thread.sleep(4000); + String exportQuery = StringUtils.format( "INSERT INTO extern(%s(exportPath => '%s'))\n" @@ -253,6 +266,9 @@ public class ITSecurityBasicQuery ); securityClient.setPermissionsToRole(ROLE_1, permissions); + // Wait for a second so that the auth is synced, to avoid flakyness + Thread.sleep(1000); + String exportQuery = StringUtils.format( "INSERT INTO extern(%s(exportPath => '%s'))\n" @@ -276,4 +292,53 @@ public class ITSecurityBasicQuery Assert.assertEquals(HttpResponseStatus.ACCEPTED, statusResponseHolder.getStatus()); } + + @Test + public void testExportTaskSubmitOverlordWithPermission() throws Exception + { + // No external write permissions for s3 + List permissions = ImmutableList.of( + new ResourceAction(new Resource(".*", "DATASOURCE"), Action.READ), + new ResourceAction(new Resource("EXTERNAL", "EXTERNAL"), Action.READ), + new ResourceAction(new Resource(LocalFileExportStorageProvider.TYPE_NAME, "EXTERNAL"), Action.WRITE), + new ResourceAction(new Resource("STATE", "STATE"), Action.READ), + new ResourceAction(new Resource(".*", "DATASOURCE"), Action.WRITE) + ); + securityClient.setPermissionsToRole(ROLE_1, permissions); + + // Wait for a second so that the auth is synced, to avoid flakiness + Thread.sleep(1000); + + String task = createTaskString(); + StatusResponseHolder statusResponseHolder = overlordResourceTestClient.submitTaskAndReturnStatusWithAuth(task, USER_1, USER_1_PASSWORD); + Assert.assertEquals(HttpResponseStatus.OK, statusResponseHolder.getStatus()); + } + + @Test + public void testExportTaskSubmitOverlordWithoutPermission() throws Exception + { + // No external write permissions for s3 + List permissions = ImmutableList.of( + new ResourceAction(new Resource(".*", "DATASOURCE"), Action.READ), + new ResourceAction(new Resource("EXTERNAL", "EXTERNAL"), Action.READ), + new ResourceAction(new Resource(S3ExportStorageProvider.TYPE_NAME, "EXTERNAL"), Action.WRITE), + new ResourceAction(new Resource("STATE", "STATE"), Action.READ), + new ResourceAction(new Resource(".*", "DATASOURCE"), Action.WRITE) + ); + securityClient.setPermissionsToRole(ROLE_1, permissions); + + // Wait for a second so that the auth is synced, to avoid flakiness + Thread.sleep(1000); + + String task = createTaskString(); + StatusResponseHolder statusResponseHolder = overlordResourceTestClient.submitTaskAndReturnStatusWithAuth(task, USER_1, USER_1_PASSWORD); + Assert.assertEquals(HttpResponseStatus.FORBIDDEN, statusResponseHolder.getStatus()); + } + + private String createTaskString() throws Exception + { + String queryId = IdUtils.newTaskId(MSQControllerTask.TYPE, "external", null); + String template = AbstractIndexerTest.getResourceAsString(EXPORT_TASK); + return StringUtils.replace(template, "%%QUERY_ID%%", queryId); + } } diff --git a/integration-tests-ex/cases/src/test/resources/indexer/export_task.json b/integration-tests-ex/cases/src/test/resources/indexer/export_task.json new file mode 100644 index 00000000000..e5bfdac4af7 --- /dev/null +++ b/integration-tests-ex/cases/src/test/resources/indexer/export_task.json @@ -0,0 +1,224 @@ +{ + "type": "query_controller", + "id": "%%QUERY_ID%%", + "spec": { + "query": { + "queryType": "scan", + "dataSource": { + "type": "external", + "inputSource": { + "type": "local", + "files": [ + "/resources/data/batch_index/json/wikipedia_index_data1.json" + ] + }, + "inputFormat": { + "type": "json", + "keepNullColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "signature": [ + { + "name": "timestamp", + "type": "STRING" + }, + { + "name": "isRobot", + "type": "STRING" + }, + { + "name": "diffUrl", + "type": "STRING" + }, + { + "name": "added", + "type": "LONG" + }, + { + "name": "countryIsoCode", + "type": "STRING" + }, + { + "name": "regionName", + "type": "STRING" + }, + { + "name": "channel", + "type": "STRING" + }, + { + "name": "flags", + "type": "STRING" + }, + { + "name": "delta", + "type": "LONG" + }, + { + "name": "isUnpatrolled", + "type": "STRING" + }, + { + "name": "isNew", + "type": "STRING" + }, + { + "name": "deltaBucket", + "type": "DOUBLE" + }, + { + "name": "isMinor", + "type": "STRING" + }, + { + "name": "isAnonymous", + "type": "STRING" + }, + { + "name": "deleted", + "type": "LONG" + }, + { + "name": "cityName", + "type": "STRING" + }, + { + "name": "metroCode", + "type": "LONG" + }, + { + "name": "namespace", + "type": "STRING" + }, + { + "name": "comment", + "type": "STRING" + }, + { + "name": "page", + "type": "STRING" + }, + { + "name": "commentLength", + "type": "LONG" + }, + { + "name": "countryName", + "type": "STRING" + }, + { + "name": "user", + "type": "STRING" + }, + { + "name": "regionIsoCode", + "type": "STRING" + } + ] + }, + "intervals": { + "type": "intervals", + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" + ] + }, + "resultFormat": "compactedList", + "columns": [ + "added", + "delta", + "page" + ], + "legacy": false, + "context": { + "__exportFileFormat": "CSV", + "__resultFormat": "array", + "__user": "allowAll", + "executionMode": "async", + "finalize": false, + "finalizeAggregations": false, + "groupByEnableMultiValueUnnesting": false, + "maxNumTasks": 4, + "maxParseExceptions": 0, + "queryId": "b1491ce2-7d2a-4a7a-baa6-25a1a77135e5", + "scanSignature": "[{\"name\":\"added\",\"type\":\"LONG\"},{\"name\":\"delta\",\"type\":\"LONG\"},{\"name\":\"page\",\"type\":\"STRING\"}]", + "sqlQueryId": "b1491ce2-7d2a-4a7a-baa6-25a1a77135e5", + "sqlStringifyArrays": false, + "waitUntilSegmentsLoad": true + }, + "columnTypes": [ + "LONG", + "LONG", + "STRING" + ], + "granularity": { + "type": "all" + } + }, + "columnMappings": [ + { + "queryColumn": "page", + "outputColumn": "page" + }, + { + "queryColumn": "added", + "outputColumn": "added" + }, + { + "queryColumn": "delta", + "outputColumn": "delta" + } + ], + "destination": { + "type": "export", + "exportStorageProvider": { + "type": "local", + "exportPath": "/shared/export/" + }, + "resultFormat": "csv" + }, + "assignmentStrategy": "max", + "tuningConfig": { + "maxNumWorkers": 3, + "maxRowsInMemory": 100000, + "rowsPerSegment": 3000000 + } + }, + "sqlQuery": " INSERT INTO extern(local(exportPath => '/shared/export/'))\n AS CSV\n SELECT page, added, delta\n FROM TABLE(\n EXTERN(\n '{\"type\":\"local\",\"files\":[\"/resources/data/batch_index/json/wikipedia_index_data1.json\"]}',\n '{\"type\":\"json\"}',\n '[{\"type\":\"string\",\"name\":\"timestamp\"},{\"type\":\"string\",\"name\":\"isRobot\"},{\"type\":\"string\",\"name\":\"diffUrl\"},{\"type\":\"long\",\"name\":\"added\"},{\"type\":\"string\",\"name\":\"countryIsoCode\"},{\"type\":\"string\",\"name\":\"regionName\"},{\"type\":\"string\",\"name\":\"channel\"},{\"type\":\"string\",\"name\":\"flags\"},{\"type\":\"long\",\"name\":\"delta\"},{\"type\":\"string\",\"name\":\"isUnpatrolled\"},{\"type\":\"string\",\"name\":\"isNew\"},{\"type\":\"double\",\"name\":\"deltaBucket\"},{\"type\":\"string\",\"name\":\"isMinor\"},{\"type\":\"string\",\"name\":\"isAnonymous\"},{\"type\":\"long\",\"name\":\"deleted\"},{\"type\":\"string\",\"name\":\"cityName\"},{\"type\":\"long\",\"name\":\"metroCode\"},{\"type\":\"string\",\"name\":\"namespace\"},{\"type\":\"string\",\"name\":\"comment\"},{\"type\":\"string\",\"name\":\"page\"},{\"type\":\"long\",\"name\":\"commentLength\"},{\"type\":\"string\",\"name\":\"countryName\"},{\"type\":\"string\",\"name\":\"user\"},{\"type\":\"string\",\"name\":\"regionIsoCode\"}]'\n )\n )\n", + "sqlQueryContext": { + "__exportFileFormat": "CSV", + "finalizeAggregations": false, + "sqlQueryId": "b1491ce2-7d2a-4a7a-baa6-25a1a77135e5", + "groupByEnableMultiValueUnnesting": false, + "maxNumTasks": 4, + "waitUntilSegmentsLoad": true, + "executionMode": "async", + "__resultFormat": "array", + "sqlStringifyArrays": false, + "queryId": "b1491ce2-7d2a-4a7a-baa6-25a1a77135e5" + }, + "sqlResultsContext": { + "timeZone": "UTC", + "serializeComplexValues": true, + "stringifyArrays": false + }, + "sqlTypeNames": [ + "VARCHAR", + "BIGINT", + "BIGINT" + ], + "nativeTypeNames": [ + "STRING", + "LONG", + "LONG" + ], + "context": { + "forceTimeChunkLock": true + }, + "groupId": "%%QUERY_ID%%", + "dataSource": "__query_select", + "resource": { + "availabilityGroup": "%%QUERY_ID%%", + "requiredCapacity": 1 + } +} diff --git a/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java b/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java index 95f56ad4b75..ac4ef536b02 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java @@ -116,6 +116,22 @@ public class OverlordResourceTestClient } } + public StatusResponseHolder submitTaskAndReturnStatusWithAuth( + final String task, + final String username, + final String password + ) throws Exception + { + return httpClient.go( + new Request(HttpMethod.POST, new URL(getIndexerURL() + "task")) + .setContent( + "application/json", + StringUtils.toUtf8(task) + ).setBasicAuthentication(username, password), + StatusResponseHandler.getInstance() + ).get(); + } + public TaskStatusPlus getTaskStatus(String taskID) { try {