Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-23941 : FilterBy operator support in get_slowlog_responses API (#1793) 

Signed-off-by: Bharath Vissapragada <bharathv@apache.org>
Signed-off-by: Nick Dimiduk <ndimiduk@apache.org>
This commit is contained in:
Viraj Jasani 2020-05-30 12:59:59 +05:30
parent aca7f6fcf3
commit b2d24a1812
No known key found for this signature in database
GPG Key ID: 3AE697641452FC5D
11 changed files with 287 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hbase-client/src/main/java/org/apache/hadoop/hbase/client/LogQueryFilter.java
View File

@ -37,12 +37,18 @@ public class LogQueryFilter {
private String userName;
private int limit = 10;
private Type type = Type.SLOW_LOG;
private FilterByOperator filterByOperator = FilterByOperator.OR;
public enum Type {
SLOW_LOG,
LARGE_LOG
}
public enum FilterByOperator {
AND,
OR
}
public String getRegionName() {
return regionName;
}
@ -91,6 +97,14 @@ public class LogQueryFilter {
this.type = type;
}
public FilterByOperator getFilterByOperator() {
return filterByOperator;
}
public void setFilterByOperator(FilterByOperator filterByOperator) {
this.filterByOperator = filterByOperator;
}
@Override
public boolean equals(Object o) {
if (this == o) {
@ -110,6 +124,7 @@ public class LogQueryFilter {
.append(tableName, that.tableName)
.append(userName, that.userName)
.append(type, that.type)
.append(filterByOperator, that.filterByOperator)
.isEquals();
}
@ -122,6 +137,7 @@ public class LogQueryFilter {
.append(userName)
.append(limit)
.append(type)
.append(filterByOperator)
.toHashCode();
}
@ -134,6 +150,7 @@ public class LogQueryFilter {
.append("userName", userName)
.append("limit", limit)
.append("type", type)
.append("filterByOperator", filterByOperator)
.toString();
}
}

5
hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
View File

@ -3890,8 +3890,7 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
@Override
public CompletableFuture<List<OnlineLogRecord>> getSlowLogResponses(
@Nullable final Set<ServerName> serverNames,
final LogQueryFilter logQueryFilter) {
@Nullable final Set<ServerName> serverNames, final LogQueryFilter logQueryFilter) {
if (CollectionUtils.isEmpty(serverNames)) {
return CompletableFuture.completedFuture(Collections.emptyList());
}
@ -3914,7 +3913,7 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
}
private CompletableFuture<List<OnlineLogRecord>> getLargeLogResponseFromServer(
final ServerName serverName, final LogQueryFilter logQueryFilter) {
final ServerName serverName, final LogQueryFilter logQueryFilter) {
return this.<List<OnlineLogRecord>>newAdminCaller()
.action((controller, stub) -> this
.adminCall(

6
hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
View File

@ -1955,6 +1955,12 @@ public final class RequestConverter {
if (StringUtils.isNotEmpty(userName)) {
builder.setUserName(userName);
}
LogQueryFilter.FilterByOperator filterByOperator = logQueryFilter.getFilterByOperator();
if (LogQueryFilter.FilterByOperator.AND.equals(filterByOperator)) {
builder.setFilterByOperator(SlowLogResponseRequest.FilterByOperator.AND);
} else {
builder.setFilterByOperator(SlowLogResponseRequest.FilterByOperator.OR);
}
return builder.setLimit(logQueryFilter.getLimit()).build();
}

6
hbase-protocol-shaded/src/main/protobuf/Admin.proto
View File

@ -282,11 +282,17 @@ message ExecuteProceduresResponse {
}
message SlowLogResponseRequest {
enum FilterByOperator {
AND = 0;
OR = 1;
}
optional string region_name = 1;
optional string table_name = 2;
optional string client_address = 3;
optional string user_name = 4;
optional uint32 limit = 5 [default = 10];
optional FilterByOperator filter_by_operator = 6 [default = OR];
}
message SlowLogResponses {

57
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/slowlog/LogEventHandler.java
View File

@ -209,7 +209,7 @@ class LogEventHandler implements EventHandler<RingBufferEnvelope> {
// latest slow logs first, operator is interested in latest records from in-memory buffer
Collections.reverse(slowLogPayloadList);
return getFilteredLogs(request, slowLogPayloadList);
return LogHandlerUtils.getFilteredLogs(request, slowLogPayloadList);
}
/**
@ -228,60 +228,7 @@ class LogEventHandler implements EventHandler<RingBufferEnvelope> {
// latest large logs first, operator is interested in latest records from in-memory buffer
Collections.reverse(slowLogPayloadList);
return getFilteredLogs(request, slowLogPayloadList);
}
private List<SlowLogPayload> getFilteredLogs(AdminProtos.SlowLogResponseRequest request,
List<SlowLogPayload> logPayloadList) {
if (isFilterProvided(request)) {
logPayloadList = filterLogs(request, logPayloadList);
}
int limit = Math.min(request.getLimit(), logPayloadList.size());
return logPayloadList.subList(0, limit);
}
private boolean isFilterProvided(AdminProtos.SlowLogResponseRequest request) {
if (StringUtils.isNotEmpty(request.getUserName())) {
return true;
}
if (StringUtils.isNotEmpty(request.getTableName())) {
return true;
}
if (StringUtils.isNotEmpty(request.getClientAddress())) {
return true;
}
return StringUtils.isNotEmpty(request.getRegionName());
}
private List<SlowLogPayload> filterLogs(AdminProtos.SlowLogResponseRequest request,
List<SlowLogPayload> slowLogPayloadList) {
List<SlowLogPayload> filteredSlowLogPayloads = new ArrayList<>();
for (SlowLogPayload slowLogPayload : slowLogPayloadList) {
if (StringUtils.isNotEmpty(request.getRegionName())) {
if (slowLogPayload.getRegionName().equals(request.getRegionName())) {
filteredSlowLogPayloads.add(slowLogPayload);
continue;
}
}
if (StringUtils.isNotEmpty(request.getTableName())) {
if (slowLogPayload.getRegionName().startsWith(request.getTableName())) {
filteredSlowLogPayloads.add(slowLogPayload);
continue;
}
}
if (StringUtils.isNotEmpty(request.getClientAddress())) {
if (slowLogPayload.getClientAddress().equals(request.getClientAddress())) {
filteredSlowLogPayloads.add(slowLogPayload);
continue;
}
}
if (StringUtils.isNotEmpty(request.getUserName())) {
if (slowLogPayload.getUserName().equals(request.getUserName())) {
filteredSlowLogPayloads.add(slowLogPayload);
}
}
}
return filteredSlowLogPayloads;
return LogHandlerUtils.getFilteredLogs(request, slowLogPayloadList);
}
/**

104
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/slowlog/LogHandlerUtils.java Normal file
View File

@ -0,0 +1,104 @@
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.regionserver.slowlog;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.TooSlowLog;
import org.apache.yetus.audience.InterfaceAudience;
/**
* Event Handler utility class
*/
@InterfaceAudience.Private
class LogHandlerUtils {
private static int getTotalFiltersCount(AdminProtos.SlowLogResponseRequest request) {
int totalFilters = 0;
if (StringUtils.isNotEmpty(request.getRegionName())) {
totalFilters++;
}
if (StringUtils.isNotEmpty(request.getTableName())) {
totalFilters++;
}
if (StringUtils.isNotEmpty(request.getClientAddress())) {
totalFilters++;
}
if (StringUtils.isNotEmpty(request.getUserName())) {
totalFilters++;
}
return totalFilters;
}
private static List<TooSlowLog.SlowLogPayload> filterLogs(
AdminProtos.SlowLogResponseRequest request,
List<TooSlowLog.SlowLogPayload> slowLogPayloadList, int totalFilters) {
List<TooSlowLog.SlowLogPayload> filteredSlowLogPayloads = new ArrayList<>();
final String regionName =
StringUtils.isNotEmpty(request.getRegionName()) ? request.getRegionName() : null;
final String tableName =
StringUtils.isNotEmpty(request.getTableName()) ? request.getTableName() : null;
final String clientAddress =
StringUtils.isNotEmpty(request.getClientAddress()) ? request.getClientAddress() : null;
final String userName =
StringUtils.isNotEmpty(request.getUserName()) ? request.getUserName() : null;
for (TooSlowLog.SlowLogPayload slowLogPayload : slowLogPayloadList) {
int totalFilterMatches = 0;
if (slowLogPayload.getRegionName().equals(regionName)) {
totalFilterMatches++;
}
if (tableName != null && slowLogPayload.getRegionName().startsWith(tableName)) {
totalFilterMatches++;
}
if (slowLogPayload.getClientAddress().equals(clientAddress)) {
totalFilterMatches++;
}
if (slowLogPayload.getUserName().equals(userName)) {
totalFilterMatches++;
}
if (request.hasFilterByOperator() && request.getFilterByOperator()
.equals(AdminProtos.SlowLogResponseRequest.FilterByOperator.AND)) {
// Filter by AND operator
if (totalFilterMatches == totalFilters) {
filteredSlowLogPayloads.add(slowLogPayload);
}
} else {
// Filter by OR operator
if (totalFilterMatches > 0) {
filteredSlowLogPayloads.add(slowLogPayload);
}
}
}
return filteredSlowLogPayloads;
}
static List<TooSlowLog.SlowLogPayload> getFilteredLogs(
AdminProtos.SlowLogResponseRequest request, List<TooSlowLog.SlowLogPayload> logPayloadList) {
int totalFilters = getTotalFiltersCount(request);
if (totalFilters > 0) {
logPayloadList = filterLogs(request, logPayloadList, totalFilters);
}
int limit = Math.min(request.getLimit(), logPayloadList.size());
return logPayloadList.subList(0, limit);
}
}

65
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/slowlog/TestSlowLogRecorder.java
View File

@ -486,6 +486,71 @@ public class TestSlowLogRecorder {
}
@Test
public void testSlowLogMixedFilters() throws Exception {
Configuration conf = applySlowLogRecorderConf(30);
slowLogRecorder = new SlowLogRecorder(conf);
AdminProtos.SlowLogResponseRequest request =
AdminProtos.SlowLogResponseRequest.newBuilder()
.setLimit(15)
.setUserName("userName_87")
.setClientAddress("client_88")
.build();
Assert.assertEquals(slowLogRecorder.getSlowLogPayloads(request).size(), 0);
for (int i = 0; i < 100; i++) {
RpcLogDetails rpcLogDetails =
getRpcLogDetails("userName_" + (i + 1), "client_" + (i + 1), "class_" + (i + 1));
slowLogRecorder.addSlowLogPayload(rpcLogDetails);
}
Assert.assertNotEquals(-1, HBASE_TESTING_UTILITY.waitFor(3000,
() -> slowLogRecorder.getSlowLogPayloads(request).size() == 2));
AdminProtos.SlowLogResponseRequest request2 = AdminProtos.SlowLogResponseRequest.newBuilder()
.setLimit(15)
.setUserName("userName_1")
.setClientAddress("client_2")
.build();
Assert.assertEquals(0, slowLogRecorder.getSlowLogPayloads(request2).size());
AdminProtos.SlowLogResponseRequest request3 =
AdminProtos.SlowLogResponseRequest.newBuilder()
.setLimit(15)
.setUserName("userName_87")
.setClientAddress("client_88")
.setFilterByOperator(AdminProtos.SlowLogResponseRequest.FilterByOperator.AND)
.build();
Assert.assertEquals(0, slowLogRecorder.getSlowLogPayloads(request3).size());
AdminProtos.SlowLogResponseRequest request4 =
AdminProtos.SlowLogResponseRequest.newBuilder()
.setLimit(15)
.setUserName("userName_87")
.setClientAddress("client_87")
.setFilterByOperator(AdminProtos.SlowLogResponseRequest.FilterByOperator.AND)
.build();
Assert.assertEquals(1, slowLogRecorder.getSlowLogPayloads(request4).size());
AdminProtos.SlowLogResponseRequest request5 =
AdminProtos.SlowLogResponseRequest.newBuilder()
.setLimit(15)
.setUserName("userName_88")
.setClientAddress("client_89")
.setFilterByOperator(AdminProtos.SlowLogResponseRequest.FilterByOperator.OR)
.build();
Assert.assertEquals(2, slowLogRecorder.getSlowLogPayloads(request5).size());
AdminProtos.SlowLogResponseRequest requestSlowLog =
AdminProtos.SlowLogResponseRequest.newBuilder()
.setLimit(15)
.build();
Assert.assertNotEquals(-1, HBASE_TESTING_UTILITY.waitFor(3000,
() -> slowLogRecorder.getSlowLogPayloads(requestSlowLog).size() == 15));
}
static RpcLogDetails getRpcLogDetails(String userName, String clientAddress, String className) {
RpcCall rpcCall = getRpcCall(userName);
return new RpcLogDetails(rpcCall, rpcCall.getParam(), clientAddress, 0, className, true, true);

10
hbase-shell/src/main/ruby/hbase/admin.rb
View File

@ -1480,6 +1480,16 @@ module Hbase
limit = args['LIMIT']
filter_params.setLimit(limit)
end
if args.key? 'FILTER_BY_OP'
filter_by_op = args['FILTER_BY_OP']
if filter_by_op != 'OR' && filter_by_op != 'AND'
raise(ArgumentError, "FILTER_BY_OP should be either OR / AND")
end
if filter_by_op == 'AND'
filter_params.setFilterByOperator(
org.apache.hadoop.hbase.client.LogQueryFilter::FilterByOperator::AND)
end
end
filter_params
end

20
hbase-shell/src/main/ruby/shell/commands/get_largelog_responses.rb
View File

@ -55,6 +55,26 @@ Examples:
=> get largelog responses that match either
provided client IP address or user name
All of above queries with filters have default OR operation applied i.e. all
records with any of the provided filters applied will be returned. However,
we can also apply AND operator i.e. all records that match all (not any) of
the provided filters should be returned.
hbase> get_largelog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'AND'}
=> get largelog responses with given region name
and table name, both should match
hbase> get_largelog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'OR'}
=> get largelog responses with given region name
or table name, any one can match
hbase> get_largelog_responses '*', {'TABLE_NAME' => 't1', 'CLIENT_IP' => '192.163.41.53:52781', 'FILTER_BY_OP' => 'AND'}
=> get largelog responses with given region name
and client IP address, both should match
Since OR is the default filter operator, without providing 'FILTER_BY_OP', query will have
same result as providing 'FILTER_BY_OP' => 'OR'.
Sometimes output can be long pretty printed json for user to scroll in
a single screen and hence user might prefer
redirecting output of get_largelog_responses to a file.

20
hbase-shell/src/main/ruby/shell/commands/get_slowlog_responses.rb
View File

@ -55,6 +55,26 @@ Examples:
=> get slowlog responses that match either
provided client IP address or user name
All of above queries with filters have default OR operation applied i.e. all
records with any of the provided filters applied will be returned. However,
we can also apply AND operator i.e. all records that match all (not any) of
the provided filters should be returned.
hbase> get_slowlog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'AND'}
=> get slowlog responses with given region name
and table name, both should match
hbase> get_slowlog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'OR'}
=> get slowlog responses with given region name
or table name, any one can match
hbase> get_slowlog_responses '*', {'TABLE_NAME' => 't1', 'CLIENT_IP' => '192.163.41.53:52781', 'FILTER_BY_OP' => 'AND'}
=> get slowlog responses with given region name
and client IP address, both should match
Since OR is the default filter operator, without providing 'FILTER_BY_OP', query will have
same result as providing 'FILTER_BY_OP' => 'OR'.
Sometimes output can be long pretty printed json for user to scroll in
a single screen and hence user might prefer
redirecting output of get_slowlog_responses to a file.

35
src/main/asciidoc/_chapters/ops_mgt.adoc
View File

@ -1899,6 +1899,29 @@ Examples:
----
All of above queries with filters have default OR operation applied i.e. all
records with any of the provided filters applied will be returned. However,
we can also apply AND operator i.e. all records that match all (not any) of
the provided filters should be returned.
----
hbase> get_slowlog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'AND'}
=> get slowlog responses with given region name
and table name, both should match
hbase> get_slowlog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'OR'}
=> get slowlog responses with given region name
or table name, any one can match
hbase> get_slowlog_responses '*', {'TABLE_NAME' => 't1', 'CLIENT_IP' => '192.163.41.53:52781', 'FILTER_BY_OP' => 'AND'}
=> get slowlog responses with given region name
and client IP address, both should match
----
Since OR is the default filter operator, without providing 'FILTER_BY_OP', query will have
same result as providing 'FILTER_BY_OP' => 'OR'.
Sometimes output can be long pretty printed json for user to scroll in
a single screen and hence user might prefer
redirecting output of get_slowlog_responses to a file.
@ -1934,6 +1957,18 @@ larger in size.
=> get largelog responses that match either
provided client IP address or user name
hbase> get_largelog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'AND'}
=> get largelog responses with given region name
and table name, both should match
hbase> get_largelog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'OR'}
=> get largelog responses with given region name
or table name, any one can match
hbase> get_largelog_responses '*', {'TABLE_NAME' => 't1', 'CLIENT_IP' => '192.163.41.53:52781', 'FILTER_BY_OP' => 'AND'}
=> get largelog responses with given region name
and client IP address, both should match
----