Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add security ITs for sending tasks to overlord (#16131) 

* Add security ITs for sending tasks to overlord

* Add security ITs for sending tasks to overlord

* Resolve test flakiness
This commit is contained in:
Adarsh Sanjeev 2024-03-18 09:33:40 +05:30 committed by GitHub
parent 1682d4570d
commit 86a24012a6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 315 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/revised-its.yml vendored
View File

@ -50,7 +50,7 @@ jobs:
matrix:
#jdk: [8, 11, 17]
jdk: [8]
it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat]
it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat, Security]
#indexer: [indexer, middleManager]
indexer: [middleManager]
uses: ./.github/workflows/reusable-revised-its.yml

9
integration-tests-ex/cases/pom.xml
View File

@ -530,5 +530,14 @@
</plugins>
</build>
</profile>
<profile>
<id>IT-Security</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<it.category>Security</it.category>
</properties>
</profile>
</profiles>
</project>

65
integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/auth/ITSecurityBasicQuery.java
View File

@ -22,8 +22,10 @@ package org.apache.druid.testsEx.auth;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Inject;
import org.apache.druid.common.utils.IdUtils;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.http.client.response.StatusResponseHolder;
import org.apache.druid.msq.indexing.MSQControllerTask;
import org.apache.druid.server.security.Action;
import org.apache.druid.server.security.Resource;
import org.apache.druid.server.security.ResourceAction;
@ -31,9 +33,11 @@ import org.apache.druid.sql.http.SqlQuery;
import org.apache.druid.storage.local.LocalFileExportStorageProvider;
import org.apache.druid.storage.s3.output.S3ExportStorageProvider;
import org.apache.druid.testing.clients.CoordinatorResourceTestClient;
import org.apache.druid.testing.clients.OverlordResourceTestClient;
import org.apache.druid.testing.clients.SecurityClient;
import org.apache.druid.testing.utils.DataLoaderHelper;
import org.apache.druid.testing.utils.MsqTestQueryHelper;
import org.apache.druid.tests.indexer.AbstractIndexerTest;
import org.apache.druid.testsEx.categories.Security;
import org.apache.druid.testsEx.config.DruidTestRunner;
import org.jboss.netty.handler.codec.http.HttpResponseStatus;
@ -62,10 +66,13 @@ public class ITSecurityBasicQuery
private CoordinatorResourceTestClient coordinatorClient;
@Inject
private SecurityClient securityClient;
@Inject
private OverlordResourceTestClient overlordResourceTestClient;
public static final String USER_1 = "user1";
public static final String ROLE_1 = "role1";
public static final String USER_1_PASSWORD = "password1";
private static final String EXPORT_TASK = "/indexer/export_task.json";
@Before
public void setUp() throws IOException
@ -154,6 +161,9 @@ public class ITSecurityBasicQuery
);
securityClient.setPermissionsToRole(ROLE_1, permissions);
// Wait for a second so that the auth is synced, to avoid flakiness
Thread.sleep(1000);
String queryLocal =
StringUtils.format(
"INSERT INTO %s\n"
@ -216,6 +226,9 @@ public class ITSecurityBasicQuery
);
securityClient.setPermissionsToRole(ROLE_1, permissions);
// Wait for a second so that the auth is synced, to avoid flakiness
Thread.sleep(4000);
String exportQuery =
StringUtils.format(
"INSERT INTO extern(%s(exportPath => '%s'))\n"
@ -253,6 +266,9 @@ public class ITSecurityBasicQuery
);
securityClient.setPermissionsToRole(ROLE_1, permissions);
// Wait for a second so that the auth is synced, to avoid flakyness
Thread.sleep(1000);
String exportQuery =
StringUtils.format(
"INSERT INTO extern(%s(exportPath => '%s'))\n"
@ -276,4 +292,53 @@ public class ITSecurityBasicQuery
Assert.assertEquals(HttpResponseStatus.ACCEPTED, statusResponseHolder.getStatus());
}
@Test
public void testExportTaskSubmitOverlordWithPermission() throws Exception
{
// No external write permissions for s3
List<ResourceAction> permissions = ImmutableList.of(
new ResourceAction(new Resource(".*", "DATASOURCE"), Action.READ),
new ResourceAction(new Resource("EXTERNAL", "EXTERNAL"), Action.READ),
new ResourceAction(new Resource(LocalFileExportStorageProvider.TYPE_NAME, "EXTERNAL"), Action.WRITE),
new ResourceAction(new Resource("STATE", "STATE"), Action.READ),
new ResourceAction(new Resource(".*", "DATASOURCE"), Action.WRITE)
);
securityClient.setPermissionsToRole(ROLE_1, permissions);
// Wait for a second so that the auth is synced, to avoid flakiness
Thread.sleep(1000);
String task = createTaskString();
StatusResponseHolder statusResponseHolder = overlordResourceTestClient.submitTaskAndReturnStatusWithAuth(task, USER_1, USER_1_PASSWORD);
Assert.assertEquals(HttpResponseStatus.OK, statusResponseHolder.getStatus());
}
@Test
public void testExportTaskSubmitOverlordWithoutPermission() throws Exception
{
// No external write permissions for s3
List<ResourceAction> permissions = ImmutableList.of(
new ResourceAction(new Resource(".*", "DATASOURCE"), Action.READ),
new ResourceAction(new Resource("EXTERNAL", "EXTERNAL"), Action.READ),
new ResourceAction(new Resource(S3ExportStorageProvider.TYPE_NAME, "EXTERNAL"), Action.WRITE),
new ResourceAction(new Resource("STATE", "STATE"), Action.READ),
new ResourceAction(new Resource(".*", "DATASOURCE"), Action.WRITE)
);
securityClient.setPermissionsToRole(ROLE_1, permissions);
// Wait for a second so that the auth is synced, to avoid flakiness
Thread.sleep(1000);
String task = createTaskString();
StatusResponseHolder statusResponseHolder = overlordResourceTestClient.submitTaskAndReturnStatusWithAuth(task, USER_1, USER_1_PASSWORD);
Assert.assertEquals(HttpResponseStatus.FORBIDDEN, statusResponseHolder.getStatus());
}
private String createTaskString() throws Exception
{
String queryId = IdUtils.newTaskId(MSQControllerTask.TYPE, "external", null);
String template = AbstractIndexerTest.getResourceAsString(EXPORT_TASK);
return StringUtils.replace(template, "%%QUERY_ID%%", queryId);
}
}

224
integration-tests-ex/cases/src/test/resources/indexer/export_task.json Normal file
View File

@ -0,0 +1,224 @@
{
"type": "query_controller",
"id": "%%QUERY_ID%%",
"spec": {
"query": {
"queryType": "scan",
"dataSource": {
"type": "external",
"inputSource": {
"type": "local",
"files": [
"/resources/data/batch_index/json/wikipedia_index_data1.json"
]
},
"inputFormat": {
"type": "json",
"keepNullColumns": false,
"assumeNewlineDelimited": false,
"useJsonNodeReader": false
},
"signature": [
{
"name": "timestamp",
"type": "STRING"
},
{
"name": "isRobot",
"type": "STRING"
},
{
"name": "diffUrl",
"type": "STRING"
},
{
"name": "added",
"type": "LONG"
},
{
"name": "countryIsoCode",
"type": "STRING"
},
{
"name": "regionName",
"type": "STRING"
},
{
"name": "channel",
"type": "STRING"
},
{
"name": "flags",
"type": "STRING"
},
{
"name": "delta",
"type": "LONG"
},
{
"name": "isUnpatrolled",
"type": "STRING"
},
{
"name": "isNew",
"type": "STRING"
},
{
"name": "deltaBucket",
"type": "DOUBLE"
},
{
"name": "isMinor",
"type": "STRING"
},
{
"name": "isAnonymous",
"type": "STRING"
},
{
"name": "deleted",
"type": "LONG"
},
{
"name": "cityName",
"type": "STRING"
},
{
"name": "metroCode",
"type": "LONG"
},
{
"name": "namespace",
"type": "STRING"
},
{
"name": "comment",
"type": "STRING"
},
{
"name": "page",
"type": "STRING"
},
{
"name": "commentLength",
"type": "LONG"
},
{
"name": "countryName",
"type": "STRING"
},
{
"name": "user",
"type": "STRING"
},
{
"name": "regionIsoCode",
"type": "STRING"
}
]
},
"intervals": {
"type": "intervals",
"intervals": [
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z"
]
},
"resultFormat": "compactedList",
"columns": [
"added",
"delta",
"page"
],
"legacy": false,
"context": {
"__exportFileFormat": "CSV",
"__resultFormat": "array",
"__user": "allowAll",
"executionMode": "async",
"finalize": false,
"finalizeAggregations": false,
"groupByEnableMultiValueUnnesting": false,
"maxNumTasks": 4,
"maxParseExceptions": 0,
"queryId": "b1491ce2-7d2a-4a7a-baa6-25a1a77135e5",
"scanSignature": "[{\"name\":\"added\",\"type\":\"LONG\"},{\"name\":\"delta\",\"type\":\"LONG\"},{\"name\":\"page\",\"type\":\"STRING\"}]",
"sqlQueryId": "b1491ce2-7d2a-4a7a-baa6-25a1a77135e5",
"sqlStringifyArrays": false,
"waitUntilSegmentsLoad": true
},
"columnTypes": [
"LONG",
"LONG",
"STRING"
],
"granularity": {
"type": "all"
}
},
"columnMappings": [
{
"queryColumn": "page",
"outputColumn": "page"
},
{
"queryColumn": "added",
"outputColumn": "added"
},
{
"queryColumn": "delta",
"outputColumn": "delta"
}
],
"destination": {
"type": "export",
"exportStorageProvider": {
"type": "local",
"exportPath": "/shared/export/"
},
"resultFormat": "csv"
},
"assignmentStrategy": "max",
"tuningConfig": {
"maxNumWorkers": 3,
"maxRowsInMemory": 100000,
"rowsPerSegment": 3000000
}
},
"sqlQuery": " INSERT INTO extern(local(exportPath => '/shared/export/'))\n AS CSV\n SELECT page, added, delta\n FROM TABLE(\n EXTERN(\n '{\"type\":\"local\",\"files\":[\"/resources/data/batch_index/json/wikipedia_index_data1.json\"]}',\n '{\"type\":\"json\"}',\n '[{\"type\":\"string\",\"name\":\"timestamp\"},{\"type\":\"string\",\"name\":\"isRobot\"},{\"type\":\"string\",\"name\":\"diffUrl\"},{\"type\":\"long\",\"name\":\"added\"},{\"type\":\"string\",\"name\":\"countryIsoCode\"},{\"type\":\"string\",\"name\":\"regionName\"},{\"type\":\"string\",\"name\":\"channel\"},{\"type\":\"string\",\"name\":\"flags\"},{\"type\":\"long\",\"name\":\"delta\"},{\"type\":\"string\",\"name\":\"isUnpatrolled\"},{\"type\":\"string\",\"name\":\"isNew\"},{\"type\":\"double\",\"name\":\"deltaBucket\"},{\"type\":\"string\",\"name\":\"isMinor\"},{\"type\":\"string\",\"name\":\"isAnonymous\"},{\"type\":\"long\",\"name\":\"deleted\"},{\"type\":\"string\",\"name\":\"cityName\"},{\"type\":\"long\",\"name\":\"metroCode\"},{\"type\":\"string\",\"name\":\"namespace\"},{\"type\":\"string\",\"name\":\"comment\"},{\"type\":\"string\",\"name\":\"page\"},{\"type\":\"long\",\"name\":\"commentLength\"},{\"type\":\"string\",\"name\":\"countryName\"},{\"type\":\"string\",\"name\":\"user\"},{\"type\":\"string\",\"name\":\"regionIsoCode\"}]'\n )\n )\n",
"sqlQueryContext": {
"__exportFileFormat": "CSV",
"finalizeAggregations": false,
"sqlQueryId": "b1491ce2-7d2a-4a7a-baa6-25a1a77135e5",
"groupByEnableMultiValueUnnesting": false,
"maxNumTasks": 4,
"waitUntilSegmentsLoad": true,
"executionMode": "async",
"__resultFormat": "array",
"sqlStringifyArrays": false,
"queryId": "b1491ce2-7d2a-4a7a-baa6-25a1a77135e5"
},
"sqlResultsContext": {
"timeZone": "UTC",
"serializeComplexValues": true,
"stringifyArrays": false
},
"sqlTypeNames": [
"VARCHAR",
"BIGINT",
"BIGINT"
],
"nativeTypeNames": [
"STRING",
"LONG",
"LONG"
],
"context": {
"forceTimeChunkLock": true
},
"groupId": "%%QUERY_ID%%",
"dataSource": "__query_select",
"resource": {
"availabilityGroup": "%%QUERY_ID%%",
"requiredCapacity": 1
}
}

16
integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java
View File

@ -116,6 +116,22 @@ public class OverlordResourceTestClient
}
}
public StatusResponseHolder submitTaskAndReturnStatusWithAuth(
final String task,
final String username,
final String password
) throws Exception
{
return httpClient.go(
new Request(HttpMethod.POST, new URL(getIndexerURL() + "task"))
.setContent(
"application/json",
StringUtils.toUtf8(task)
).setBasicAuthentication(username, password),
StatusResponseHandler.getInstance()
).get();
}
public TaskStatusPlus getTaskStatus(String taskID)
{
try {