diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/LogQueryFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/LogQueryFilter.java index 8734f7b48d5..728aba4a60b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/LogQueryFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/LogQueryFilter.java @@ -37,12 +37,18 @@ public class LogQueryFilter { private String userName; private int limit = 10; private Type type = Type.SLOW_LOG; + private FilterByOperator filterByOperator = FilterByOperator.OR; public enum Type { SLOW_LOG, LARGE_LOG } + public enum FilterByOperator { + AND, + OR + } + public String getRegionName() { return regionName; } @@ -91,6 +97,14 @@ public class LogQueryFilter { this.type = type; } + public FilterByOperator getFilterByOperator() { + return filterByOperator; + } + + public void setFilterByOperator(FilterByOperator filterByOperator) { + this.filterByOperator = filterByOperator; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -110,6 +124,7 @@ public class LogQueryFilter { .append(tableName, that.tableName) .append(userName, that.userName) .append(type, that.type) + .append(filterByOperator, that.filterByOperator) .isEquals(); } @@ -122,6 +137,7 @@ public class LogQueryFilter { .append(userName) .append(limit) .append(type) + .append(filterByOperator) .toHashCode(); } @@ -134,6 +150,7 @@ public class LogQueryFilter { .append("userName", userName) .append("limit", limit) .append("type", type) + .append("filterByOperator", filterByOperator) .toString(); } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index d98b02e1718..9792c9f2292 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -3890,8 +3890,7 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { @Override public CompletableFuture> getSlowLogResponses( - @Nullable final Set serverNames, - final LogQueryFilter logQueryFilter) { + @Nullable final Set serverNames, final LogQueryFilter logQueryFilter) { if (CollectionUtils.isEmpty(serverNames)) { return CompletableFuture.completedFuture(Collections.emptyList()); } @@ -3914,7 +3913,7 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { } private CompletableFuture> getLargeLogResponseFromServer( - final ServerName serverName, final LogQueryFilter logQueryFilter) { + final ServerName serverName, final LogQueryFilter logQueryFilter) { return this.>newAdminCaller() .action((controller, stub) -> this .adminCall( diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java index 63ef6f41228..00f20b44c8f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java @@ -1955,6 +1955,12 @@ public final class RequestConverter { if (StringUtils.isNotEmpty(userName)) { builder.setUserName(userName); } + LogQueryFilter.FilterByOperator filterByOperator = logQueryFilter.getFilterByOperator(); + if (LogQueryFilter.FilterByOperator.AND.equals(filterByOperator)) { + builder.setFilterByOperator(SlowLogResponseRequest.FilterByOperator.AND); + } else { + builder.setFilterByOperator(SlowLogResponseRequest.FilterByOperator.OR); + } return builder.setLimit(logQueryFilter.getLimit()).build(); } diff --git a/hbase-protocol-shaded/src/main/protobuf/Admin.proto b/hbase-protocol-shaded/src/main/protobuf/Admin.proto index 8dd55162783..a9fb63ba80b 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Admin.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Admin.proto @@ -282,11 +282,17 @@ message ExecuteProceduresResponse { } message SlowLogResponseRequest { + enum FilterByOperator { + AND = 0; + OR = 1; + } + optional string region_name = 1; optional string table_name = 2; optional string client_address = 3; optional string user_name = 4; optional uint32 limit = 5 [default = 10]; + optional FilterByOperator filter_by_operator = 6 [default = OR]; } message SlowLogResponses { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/slowlog/LogEventHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/slowlog/LogEventHandler.java index 8d500de571c..9c147e3c21d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/slowlog/LogEventHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/slowlog/LogEventHandler.java @@ -209,7 +209,7 @@ class LogEventHandler implements EventHandler { // latest slow logs first, operator is interested in latest records from in-memory buffer Collections.reverse(slowLogPayloadList); - return getFilteredLogs(request, slowLogPayloadList); + return LogHandlerUtils.getFilteredLogs(request, slowLogPayloadList); } /** @@ -228,60 +228,7 @@ class LogEventHandler implements EventHandler { // latest large logs first, operator is interested in latest records from in-memory buffer Collections.reverse(slowLogPayloadList); - return getFilteredLogs(request, slowLogPayloadList); - } - - private List getFilteredLogs(AdminProtos.SlowLogResponseRequest request, - List logPayloadList) { - if (isFilterProvided(request)) { - logPayloadList = filterLogs(request, logPayloadList); - } - int limit = Math.min(request.getLimit(), logPayloadList.size()); - return logPayloadList.subList(0, limit); - } - - private boolean isFilterProvided(AdminProtos.SlowLogResponseRequest request) { - if (StringUtils.isNotEmpty(request.getUserName())) { - return true; - } - if (StringUtils.isNotEmpty(request.getTableName())) { - return true; - } - if (StringUtils.isNotEmpty(request.getClientAddress())) { - return true; - } - return StringUtils.isNotEmpty(request.getRegionName()); - } - - private List filterLogs(AdminProtos.SlowLogResponseRequest request, - List slowLogPayloadList) { - List filteredSlowLogPayloads = new ArrayList<>(); - for (SlowLogPayload slowLogPayload : slowLogPayloadList) { - if (StringUtils.isNotEmpty(request.getRegionName())) { - if (slowLogPayload.getRegionName().equals(request.getRegionName())) { - filteredSlowLogPayloads.add(slowLogPayload); - continue; - } - } - if (StringUtils.isNotEmpty(request.getTableName())) { - if (slowLogPayload.getRegionName().startsWith(request.getTableName())) { - filteredSlowLogPayloads.add(slowLogPayload); - continue; - } - } - if (StringUtils.isNotEmpty(request.getClientAddress())) { - if (slowLogPayload.getClientAddress().equals(request.getClientAddress())) { - filteredSlowLogPayloads.add(slowLogPayload); - continue; - } - } - if (StringUtils.isNotEmpty(request.getUserName())) { - if (slowLogPayload.getUserName().equals(request.getUserName())) { - filteredSlowLogPayloads.add(slowLogPayload); - } - } - } - return filteredSlowLogPayloads; + return LogHandlerUtils.getFilteredLogs(request, slowLogPayloadList); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/slowlog/LogHandlerUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/slowlog/LogHandlerUtils.java new file mode 100644 index 00000000000..f4d850fdb56 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/slowlog/LogHandlerUtils.java @@ -0,0 +1,104 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.slowlog; + +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.TooSlowLog; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Event Handler utility class + */ +@InterfaceAudience.Private +class LogHandlerUtils { + + private static int getTotalFiltersCount(AdminProtos.SlowLogResponseRequest request) { + int totalFilters = 0; + if (StringUtils.isNotEmpty(request.getRegionName())) { + totalFilters++; + } + if (StringUtils.isNotEmpty(request.getTableName())) { + totalFilters++; + } + if (StringUtils.isNotEmpty(request.getClientAddress())) { + totalFilters++; + } + if (StringUtils.isNotEmpty(request.getUserName())) { + totalFilters++; + } + return totalFilters; + } + + private static List filterLogs( + AdminProtos.SlowLogResponseRequest request, + List slowLogPayloadList, int totalFilters) { + List filteredSlowLogPayloads = new ArrayList<>(); + final String regionName = + StringUtils.isNotEmpty(request.getRegionName()) ? request.getRegionName() : null; + final String tableName = + StringUtils.isNotEmpty(request.getTableName()) ? request.getTableName() : null; + final String clientAddress = + StringUtils.isNotEmpty(request.getClientAddress()) ? request.getClientAddress() : null; + final String userName = + StringUtils.isNotEmpty(request.getUserName()) ? request.getUserName() : null; + for (TooSlowLog.SlowLogPayload slowLogPayload : slowLogPayloadList) { + int totalFilterMatches = 0; + if (slowLogPayload.getRegionName().equals(regionName)) { + totalFilterMatches++; + } + if (tableName != null && slowLogPayload.getRegionName().startsWith(tableName)) { + totalFilterMatches++; + } + if (slowLogPayload.getClientAddress().equals(clientAddress)) { + totalFilterMatches++; + } + if (slowLogPayload.getUserName().equals(userName)) { + totalFilterMatches++; + } + if (request.hasFilterByOperator() && request.getFilterByOperator() + .equals(AdminProtos.SlowLogResponseRequest.FilterByOperator.AND)) { + // Filter by AND operator + if (totalFilterMatches == totalFilters) { + filteredSlowLogPayloads.add(slowLogPayload); + } + } else { + // Filter by OR operator + if (totalFilterMatches > 0) { + filteredSlowLogPayloads.add(slowLogPayload); + } + } + } + return filteredSlowLogPayloads; + } + + static List getFilteredLogs( + AdminProtos.SlowLogResponseRequest request, List logPayloadList) { + int totalFilters = getTotalFiltersCount(request); + if (totalFilters > 0) { + logPayloadList = filterLogs(request, logPayloadList, totalFilters); + } + int limit = Math.min(request.getLimit(), logPayloadList.size()); + return logPayloadList.subList(0, limit); + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/slowlog/TestSlowLogRecorder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/slowlog/TestSlowLogRecorder.java index 863e27bd297..f90bbc04423 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/slowlog/TestSlowLogRecorder.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/slowlog/TestSlowLogRecorder.java @@ -486,6 +486,71 @@ public class TestSlowLogRecorder { } + @Test + public void testSlowLogMixedFilters() throws Exception { + + Configuration conf = applySlowLogRecorderConf(30); + slowLogRecorder = new SlowLogRecorder(conf); + AdminProtos.SlowLogResponseRequest request = + AdminProtos.SlowLogResponseRequest.newBuilder() + .setLimit(15) + .setUserName("userName_87") + .setClientAddress("client_88") + .build(); + + Assert.assertEquals(slowLogRecorder.getSlowLogPayloads(request).size(), 0); + + for (int i = 0; i < 100; i++) { + RpcLogDetails rpcLogDetails = + getRpcLogDetails("userName_" + (i + 1), "client_" + (i + 1), "class_" + (i + 1)); + slowLogRecorder.addSlowLogPayload(rpcLogDetails); + } + + Assert.assertNotEquals(-1, HBASE_TESTING_UTILITY.waitFor(3000, + () -> slowLogRecorder.getSlowLogPayloads(request).size() == 2)); + + AdminProtos.SlowLogResponseRequest request2 = AdminProtos.SlowLogResponseRequest.newBuilder() + .setLimit(15) + .setUserName("userName_1") + .setClientAddress("client_2") + .build(); + Assert.assertEquals(0, slowLogRecorder.getSlowLogPayloads(request2).size()); + + AdminProtos.SlowLogResponseRequest request3 = + AdminProtos.SlowLogResponseRequest.newBuilder() + .setLimit(15) + .setUserName("userName_87") + .setClientAddress("client_88") + .setFilterByOperator(AdminProtos.SlowLogResponseRequest.FilterByOperator.AND) + .build(); + Assert.assertEquals(0, slowLogRecorder.getSlowLogPayloads(request3).size()); + + AdminProtos.SlowLogResponseRequest request4 = + AdminProtos.SlowLogResponseRequest.newBuilder() + .setLimit(15) + .setUserName("userName_87") + .setClientAddress("client_87") + .setFilterByOperator(AdminProtos.SlowLogResponseRequest.FilterByOperator.AND) + .build(); + Assert.assertEquals(1, slowLogRecorder.getSlowLogPayloads(request4).size()); + + AdminProtos.SlowLogResponseRequest request5 = + AdminProtos.SlowLogResponseRequest.newBuilder() + .setLimit(15) + .setUserName("userName_88") + .setClientAddress("client_89") + .setFilterByOperator(AdminProtos.SlowLogResponseRequest.FilterByOperator.OR) + .build(); + Assert.assertEquals(2, slowLogRecorder.getSlowLogPayloads(request5).size()); + + AdminProtos.SlowLogResponseRequest requestSlowLog = + AdminProtos.SlowLogResponseRequest.newBuilder() + .setLimit(15) + .build(); + Assert.assertNotEquals(-1, HBASE_TESTING_UTILITY.waitFor(3000, + () -> slowLogRecorder.getSlowLogPayloads(requestSlowLog).size() == 15)); + } + static RpcLogDetails getRpcLogDetails(String userName, String clientAddress, String className) { RpcCall rpcCall = getRpcCall(userName); return new RpcLogDetails(rpcCall, rpcCall.getParam(), clientAddress, 0, className, true, true); diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 79cc7dabb61..b970c36be5c 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -1480,6 +1480,16 @@ module Hbase limit = args['LIMIT'] filter_params.setLimit(limit) end + if args.key? 'FILTER_BY_OP' + filter_by_op = args['FILTER_BY_OP'] + if filter_by_op != 'OR' && filter_by_op != 'AND' + raise(ArgumentError, "FILTER_BY_OP should be either OR / AND") + end + if filter_by_op == 'AND' + filter_params.setFilterByOperator( + org.apache.hadoop.hbase.client.LogQueryFilter::FilterByOperator::AND) + end + end filter_params end diff --git a/hbase-shell/src/main/ruby/shell/commands/get_largelog_responses.rb b/hbase-shell/src/main/ruby/shell/commands/get_largelog_responses.rb index 3bdf61457ec..f7effe7a6d1 100644 --- a/hbase-shell/src/main/ruby/shell/commands/get_largelog_responses.rb +++ b/hbase-shell/src/main/ruby/shell/commands/get_largelog_responses.rb @@ -55,6 +55,26 @@ Examples: => get largelog responses that match either provided client IP address or user name +All of above queries with filters have default OR operation applied i.e. all +records with any of the provided filters applied will be returned. However, +we can also apply AND operator i.e. all records that match all (not any) of +the provided filters should be returned. + + hbase> get_largelog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'AND'} + => get largelog responses with given region name + and table name, both should match + + hbase> get_largelog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'OR'} + => get largelog responses with given region name + or table name, any one can match + + hbase> get_largelog_responses '*', {'TABLE_NAME' => 't1', 'CLIENT_IP' => '192.163.41.53:52781', 'FILTER_BY_OP' => 'AND'} + => get largelog responses with given region name + and client IP address, both should match + +Since OR is the default filter operator, without providing 'FILTER_BY_OP', query will have +same result as providing 'FILTER_BY_OP' => 'OR'. + Sometimes output can be long pretty printed json for user to scroll in a single screen and hence user might prefer redirecting output of get_largelog_responses to a file. diff --git a/hbase-shell/src/main/ruby/shell/commands/get_slowlog_responses.rb b/hbase-shell/src/main/ruby/shell/commands/get_slowlog_responses.rb index 55759ca2840..23a3ec2576d 100644 --- a/hbase-shell/src/main/ruby/shell/commands/get_slowlog_responses.rb +++ b/hbase-shell/src/main/ruby/shell/commands/get_slowlog_responses.rb @@ -55,6 +55,26 @@ Examples: => get slowlog responses that match either provided client IP address or user name +All of above queries with filters have default OR operation applied i.e. all +records with any of the provided filters applied will be returned. However, +we can also apply AND operator i.e. all records that match all (not any) of +the provided filters should be returned. + + hbase> get_slowlog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'AND'} + => get slowlog responses with given region name + and table name, both should match + + hbase> get_slowlog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'OR'} + => get slowlog responses with given region name + or table name, any one can match + + hbase> get_slowlog_responses '*', {'TABLE_NAME' => 't1', 'CLIENT_IP' => '192.163.41.53:52781', 'FILTER_BY_OP' => 'AND'} + => get slowlog responses with given region name + and client IP address, both should match + +Since OR is the default filter operator, without providing 'FILTER_BY_OP', query will have +same result as providing 'FILTER_BY_OP' => 'OR'. + Sometimes output can be long pretty printed json for user to scroll in a single screen and hence user might prefer redirecting output of get_slowlog_responses to a file. diff --git a/src/main/asciidoc/_chapters/ops_mgt.adoc b/src/main/asciidoc/_chapters/ops_mgt.adoc index 74f26ca587f..6dbc41ee7e3 100644 --- a/src/main/asciidoc/_chapters/ops_mgt.adoc +++ b/src/main/asciidoc/_chapters/ops_mgt.adoc @@ -1899,6 +1899,29 @@ Examples: ---- +All of above queries with filters have default OR operation applied i.e. all +records with any of the provided filters applied will be returned. However, +we can also apply AND operator i.e. all records that match all (not any) of +the provided filters should be returned. + +---- + hbase> get_slowlog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'AND'} + => get slowlog responses with given region name + and table name, both should match + + hbase> get_slowlog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'OR'} + => get slowlog responses with given region name + or table name, any one can match + + hbase> get_slowlog_responses '*', {'TABLE_NAME' => 't1', 'CLIENT_IP' => '192.163.41.53:52781', 'FILTER_BY_OP' => 'AND'} + => get slowlog responses with given region name + and client IP address, both should match + +---- +Since OR is the default filter operator, without providing 'FILTER_BY_OP', query will have +same result as providing 'FILTER_BY_OP' => 'OR'. + + Sometimes output can be long pretty printed json for user to scroll in a single screen and hence user might prefer redirecting output of get_slowlog_responses to a file. @@ -1934,6 +1957,18 @@ larger in size. => get largelog responses that match either provided client IP address or user name + hbase> get_largelog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'AND'} + => get largelog responses with given region name + and table name, both should match + + hbase> get_largelog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'OR'} + => get largelog responses with given region name + or table name, any one can match + + hbase> get_largelog_responses '*', {'TABLE_NAME' => 't1', 'CLIENT_IP' => '192.163.41.53:52781', 'FILTER_BY_OP' => 'AND'} + => get largelog responses with given region name + and client IP address, both should match + ----