Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Deprecate task audit logging (#6368) 

* Deprecate task audit logging

* fix test

* fix it test
This commit is contained in:
Jihoon Son 2018-09-26 16:28:02 -07:00 committed by Fangjin Yang
parent c9d281a2e9
commit 6fb503c073
17 changed files with 92 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/content/operations/api-reference.md
View File

@ -368,9 +368,13 @@ Retrieve the status of a task.
Retrieve information about the segments of a task.
<div class="note caution">
This API is deprecated and will be removed in future releases.
</div>
* `/druid/indexer/v1/task/{taskId}/reports`
Retrieve a [task completion report](../ingestion/reports.html) for a task. Only works for completed tasks.
Retrieve a [task completion report](../ingestion/reports.html) for a task. Only works for completed tasks.
#### POST

4
extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java
View File

@ -63,6 +63,7 @@ import org.apache.druid.indexing.common.TestUtils;
import org.apache.druid.indexing.common.actions.LocalTaskActionClientFactory;
import org.apache.druid.indexing.common.actions.TaskActionClientFactory;
import org.apache.druid.indexing.common.actions.TaskActionToolbox;
import org.apache.druid.indexing.common.actions.TaskAuditLogConfig;
import org.apache.druid.indexing.common.config.TaskConfig;
import org.apache.druid.indexing.common.config.TaskStorageConfig;
import org.apache.druid.indexing.common.stats.RowIngestionMeters;
@ -2194,7 +2195,8 @@ public class KafkaIndexTaskTest
);
final TaskActionClientFactory taskActionClientFactory = new LocalTaskActionClientFactory(
taskStorage,
taskActionToolbox
taskActionToolbox,
new TaskAuditLogConfig(false)
);
final SegmentHandoffNotifierFactory handoffNotifierFactory = dataSource -> new SegmentHandoffNotifier()
{

11
indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LocalTaskActionClient.java
View File

@ -31,14 +31,21 @@ public class LocalTaskActionClient implements TaskActionClient
private final Task task;
private final TaskStorage storage;
private final TaskActionToolbox toolbox;
private final TaskAuditLogConfig auditLogConfig;
private static final EmittingLogger log = new EmittingLogger(LocalTaskActionClient.class);
public LocalTaskActionClient(Task task, TaskStorage storage, TaskActionToolbox toolbox)
public LocalTaskActionClient(
Task task,
TaskStorage storage,
TaskActionToolbox toolbox,
TaskAuditLogConfig auditLogConfig
)
{
this.task = task;
this.storage = storage;
this.toolbox = toolbox;
this.auditLogConfig = auditLogConfig;
}
@Override
@ -46,7 +53,7 @@ public class LocalTaskActionClient implements TaskActionClient
{
log.info("Performing action for task[%s]: %s", task.getId(), taskAction);
if (taskAction.isAudited()) {
if (auditLogConfig.isEnabled() && taskAction.isAudited()) {
// Add audit log
try {
final long auditLogStartTime = System.currentTimeMillis();

6
indexing-service/src/main/java/org/apache/druid/indexing/common/actions/LocalTaskActionClientFactory.java
View File

@ -29,17 +29,19 @@ public class LocalTaskActionClientFactory implements TaskActionClientFactory
{
private final TaskStorage storage;
private final TaskActionToolbox toolbox;
private final TaskAuditLogConfig auditLogConfig;
@Inject
public LocalTaskActionClientFactory(TaskStorage storage, TaskActionToolbox toolbox)
public LocalTaskActionClientFactory(TaskStorage storage, TaskActionToolbox toolbox, TaskAuditLogConfig auditLogConfig)
{
this.storage = storage;
this.toolbox = toolbox;
this.auditLogConfig = auditLogConfig;
}
@Override
public TaskActionClient create(Task task)
{
return new LocalTaskActionClient(task, storage, toolbox);
return new LocalTaskActionClient(task, storage, toolbox, auditLogConfig);
}
}

45
indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TaskAuditLogConfig.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.indexing.common.actions;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* The configuration for task audit logging.
* This class will be removed in future releases. See https://github.com/apache/incubator-druid/issues/5859.
*/
@Deprecated
public class TaskAuditLogConfig
{
@JsonProperty
private final boolean enabled;
@JsonCreator
public TaskAuditLogConfig(@JsonProperty("enabled") boolean enabled)
{
this.enabled = enabled;
}
@JsonProperty("enabled")
public boolean isEnabled()
{
return enabled;
}
}

2
indexing-service/src/main/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorage.java
View File

@ -373,6 +373,7 @@ public class HeapMemoryTaskStorage implements TaskStorage
}
}
@Deprecated
@Override
public <T> void addAuditLog(Task task, TaskAction<T> taskAction)
{
@ -386,6 +387,7 @@ public class HeapMemoryTaskStorage implements TaskStorage
}
}
@Deprecated
@Override
public List<TaskAction> getAuditLogs(String taskid)
{

2
indexing-service/src/main/java/org/apache/druid/indexing/overlord/MetadataTaskStorage.java
View File

@ -291,6 +291,7 @@ public class MetadataTaskStorage implements TaskStorage
);
}
@Deprecated
@Override
public <T> void addAuditLog(final Task task, final TaskAction<T> taskAction)
{
@ -301,6 +302,7 @@ public class MetadataTaskStorage implements TaskStorage
handler.addLog(task.getId(), taskAction);
}
@Deprecated
@Override
public List<TaskAction> getAuditLogs(final String taskId)
{

2
indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskStorage.java
View File

@ -106,6 +106,7 @@ public interface TaskStorage
*
* @param <T> task action return type
*/
@Deprecated
<T> void addAuditLog(Task task, TaskAction<T> taskAction);
/**
@ -114,6 +115,7 @@ public interface TaskStorage
* @param taskid task ID
* @return list of task actions
*/
@Deprecated
List<TaskAction> getAuditLogs(String taskid);
/**

1
indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskStorageQueryAdapter.java
View File

@ -93,6 +93,7 @@ public class TaskStorageQueryAdapter
* @param taskid task ID
* @return set of segments created by the specified task
*/
@Deprecated
public Set<DataSegment> getInsertedSegments(final String taskid)
{
final Set<DataSegment> segments = Sets.newHashSet();

1
indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java
View File

@ -307,6 +307,7 @@ public class OverlordResource
return Response.status(status).entity(response).build();
}
@Deprecated
@GET
@Path("/task/{taskid}/segments")
@Produces(MediaType.APPLICATION_JSON)

4
indexing-service/src/test/java/org/apache/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java
View File

@ -62,6 +62,7 @@ import org.apache.druid.indexing.common.TestUtils;
import org.apache.druid.indexing.common.actions.LocalTaskActionClientFactory;
import org.apache.druid.indexing.common.actions.TaskActionClientFactory;
import org.apache.druid.indexing.common.actions.TaskActionToolbox;
import org.apache.druid.indexing.common.actions.TaskAuditLogConfig;
import org.apache.druid.indexing.common.config.TaskConfig;
import org.apache.druid.indexing.common.config.TaskStorageConfig;
import org.apache.druid.indexing.common.index.RealtimeAppenderatorIngestionSpec;
@ -1526,7 +1527,8 @@ public class AppenderatorDriverRealtimeIndexTaskTest
);
final TaskActionClientFactory taskActionClientFactory = new LocalTaskActionClientFactory(
taskStorage,
taskActionToolbox
taskActionToolbox,
new TaskAuditLogConfig(false)
);
IntervalChunkingQueryRunnerDecorator queryRunnerDecorator = new IntervalChunkingQueryRunnerDecorator(
null,

3
indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java
View File

@ -26,6 +26,7 @@ import org.apache.druid.indexing.common.TestUtils;
import org.apache.druid.indexing.common.actions.LocalTaskActionClient;
import org.apache.druid.indexing.common.actions.TaskActionClient;
import org.apache.druid.indexing.common.actions.TaskActionToolbox;
import org.apache.druid.indexing.common.actions.TaskAuditLogConfig;
import org.apache.druid.indexing.common.config.TaskStorageConfig;
import org.apache.druid.indexing.overlord.HeapMemoryTaskStorage;
import org.apache.druid.indexing.overlord.TaskLockbox;
@ -54,7 +55,7 @@ public abstract class IngestionTestBase
public TaskActionClient createActionClient(Task task)
{
return new LocalTaskActionClient(task, taskStorage, createTaskActionToolbox());
return new LocalTaskActionClient(task, taskStorage, createTaskActionToolbox(), new TaskAuditLogConfig(false));
}
public void prepareTaskForLocking(Task task) throws EntryExistsException

4
indexing-service/src/test/java/org/apache/druid/indexing/common/task/RealtimeIndexTaskTest.java
View File

@ -55,6 +55,7 @@ import org.apache.druid.indexing.common.TestUtils;
import org.apache.druid.indexing.common.actions.LocalTaskActionClientFactory;
import org.apache.druid.indexing.common.actions.TaskActionClientFactory;
import org.apache.druid.indexing.common.actions.TaskActionToolbox;
import org.apache.druid.indexing.common.actions.TaskAuditLogConfig;
import org.apache.druid.indexing.common.config.TaskConfig;
import org.apache.druid.indexing.common.config.TaskStorageConfig;
import org.apache.druid.indexing.overlord.HeapMemoryTaskStorage;
@ -988,7 +989,8 @@ public class RealtimeIndexTaskTest
);
final TaskActionClientFactory taskActionClientFactory = new LocalTaskActionClientFactory(
taskStorage,
taskActionToolbox
taskActionToolbox,
new TaskAuditLogConfig(false)
);
IntervalChunkingQueryRunnerDecorator queryRunnerDecorator = new IntervalChunkingQueryRunnerDecorator(
null,

4
indexing-service/src/test/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java
View File

@ -47,6 +47,7 @@ import org.apache.druid.indexing.common.TaskToolboxFactory;
import org.apache.druid.indexing.common.TestUtils;
import org.apache.druid.indexing.common.actions.LocalTaskActionClientFactory;
import org.apache.druid.indexing.common.actions.TaskActionToolbox;
import org.apache.druid.indexing.common.actions.TaskAuditLogConfig;
import org.apache.druid.indexing.common.config.TaskConfig;
import org.apache.druid.indexing.common.config.TaskStorageConfig;
import org.apache.druid.indexing.common.task.NoopTask;
@ -222,7 +223,8 @@ public class IngestSegmentFirehoseFactoryTest
newMockEmitter(),
EasyMock.createMock(SupervisorManager.class),
new Counters()
)
),
new TaskAuditLogConfig(false)
);
SegmentHandoffNotifierFactory notifierFactory = EasyMock.createNiceMock(SegmentHandoffNotifierFactory.class);
EasyMock.replay(notifierFactory);

4
indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java
View File

@ -57,6 +57,7 @@ import org.apache.druid.indexing.common.actions.LockListAction;
import org.apache.druid.indexing.common.actions.SegmentInsertAction;
import org.apache.druid.indexing.common.actions.TaskActionClientFactory;
import org.apache.druid.indexing.common.actions.TaskActionToolbox;
import org.apache.druid.indexing.common.actions.TaskAuditLogConfig;
import org.apache.druid.indexing.common.config.TaskConfig;
import org.apache.druid.indexing.common.config.TaskStorageConfig;
import org.apache.druid.indexing.common.stats.RowIngestionMetersFactory;
@ -538,7 +539,8 @@ public class TaskLifecycleTest
emitter,
EasyMock.createMock(SupervisorManager.class),
new Counters()
)
),
new TaskAuditLogConfig(true)
);
File tmpDir = temporaryFolder.newFolder();
taskConfig = new TaskConfig(tmpDir.toString(), null, null, 50000, null, false, null, null);

2
services/src/main/java/org/apache/druid/cli/CliOverlord.java
View File

@ -54,6 +54,7 @@ import org.apache.druid.guice.annotations.Json;
import org.apache.druid.indexing.common.actions.LocalTaskActionClientFactory;
import org.apache.druid.indexing.common.actions.TaskActionClientFactory;
import org.apache.druid.indexing.common.actions.TaskActionToolbox;
import org.apache.druid.indexing.common.actions.TaskAuditLogConfig;
import org.apache.druid.indexing.common.config.TaskConfig;
import org.apache.druid.indexing.common.config.TaskStorageConfig;
import org.apache.druid.indexing.common.stats.DropwizardRowIngestionMetersFactory;
@ -166,6 +167,7 @@ public class CliOverlord extends ServerRunnable
JsonConfigProvider.bind(binder, "druid.coordinator.asOverlord", CoordinatorOverlordServiceConfig.class);
JsonConfigProvider.bind(binder, "druid.indexer.queue", TaskQueueConfig.class);
JsonConfigProvider.bind(binder, "druid.indexer.task", TaskConfig.class);
JsonConfigProvider.bind(binder, "druid.indexer.auditlog", TaskAuditLogConfig.class);
binder.bind(TaskMaster.class).in(ManageLifecycle.class);

2
services/src/main/java/org/apache/druid/cli/CliPeon.java
View File

@ -63,6 +63,7 @@ import org.apache.druid.indexing.common.actions.LocalTaskActionClientFactory;
import org.apache.druid.indexing.common.actions.RemoteTaskActionClientFactory;
import org.apache.druid.indexing.common.actions.TaskActionClientFactory;
import org.apache.druid.indexing.common.actions.TaskActionToolbox;
import org.apache.druid.indexing.common.actions.TaskAuditLogConfig;
import org.apache.druid.indexing.common.config.TaskConfig;
import org.apache.druid.indexing.common.config.TaskStorageConfig;
import org.apache.druid.indexing.common.stats.DropwizardRowIngestionMetersFactory;
@ -199,6 +200,7 @@ public class CliPeon extends GuiceRunnable
binder.bind(TaskToolboxFactory.class).in(LazySingleton.class);
JsonConfigProvider.bind(binder, "druid.indexer.task", TaskConfig.class);
JsonConfigProvider.bind(binder, "druid.indexer.auditlog", TaskAuditLogConfig.class);
JsonConfigProvider.bind(binder, "druid.peon.taskActionClient.retry", RetryPolicyConfig.class);
configureTaskActionClient(binder);