diff --git a/hbase-logging/src/test/java/org/apache/hadoop/hbase/logging/HBaseTestAppender.java b/hbase-logging/src/test/java/org/apache/hadoop/hbase/logging/HBaseTestAppender.java new file mode 100644 index 00000000000..6ac1ce053cd --- /dev/null +++ b/hbase-logging/src/test/java/org/apache/hadoop/hbase/logging/HBaseTestAppender.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.logging; + +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.Core; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender; +import org.apache.logging.log4j.core.appender.ManagerFactory; +import org.apache.logging.log4j.core.appender.OutputStreamManager; +import org.apache.logging.log4j.core.appender.rolling.FileSize; +import org.apache.logging.log4j.core.config.Property; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginBuilderAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory; +import org.apache.logging.log4j.core.config.plugins.validation.constraints.Required; + +/** + * Log4j2 appender to be used when running UTs. + *

+ * The main point here is to limit the total output size to prevent eating all the space of our ci + * system when something wrong in our code. + *

+ * See HBASE-26947 for more details. + */ +@Plugin(name = HBaseTestAppender.PLUGIN_NAME, category = Core.CATEGORY_NAME, + elementType = Appender.ELEMENT_TYPE, printObject = true) +public final class HBaseTestAppender extends AbstractOutputStreamAppender { + + public static final String PLUGIN_NAME = "HBaseTest"; + private static final HBaseTestManagerFactory FACTORY = new HBaseTestManagerFactory(); + + public static class Builder> extends AbstractOutputStreamAppender.Builder + implements org.apache.logging.log4j.core.util.Builder { + + @PluginBuilderAttribute + @Required + private Target target; + + @PluginBuilderAttribute + @Required + private String maxSize; + + public B setTarget(Target target) { + this.target = target; + return asBuilder(); + } + + public B setMaxSize(String maxSize) { + this.maxSize = maxSize; + return asBuilder(); + } + + @Override + public HBaseTestAppender build() { + long size = FileSize.parse(maxSize, -1); + if (size <= 0) { + LOGGER.error("Invalid maxSize {}", size); + } + Layout layout = getOrCreateLayout(StandardCharsets.UTF_8); + OutputStreamManager manager = + OutputStreamManager.getManager(target.name(), FACTORY, new FactoryData(target, layout)); + return new HBaseTestAppender(getName(), + layout, + getFilter(), + isIgnoreExceptions(), + isImmediateFlush(), + getPropertyArray(), + manager, + size); + } + } + + /** + * Data to pass to factory method.Unable to instantiate + */ + private static class FactoryData { + private final Target target; + private final Layout layout; + + public FactoryData(Target target, Layout layout) { + this.target = target; + this.layout = layout; + } + } + + /** + * Factory to create the Appender. + */ + private static class HBaseTestManagerFactory + implements ManagerFactory { + + /** + * Create an OutputStreamManager. + * @param name The name of the entity to manage. + * @param data The data required to create the entity. + * @return The OutputStreamManager + */ + @Override + public HBaseTestOutputStreamManager createManager(final String name, final FactoryData data) { + return new HBaseTestOutputStreamManager(data.target, data.layout); + } + } + + @PluginBuilderFactory + public static > B newBuilder() { + return new Builder().asBuilder(); + } + + private final long maxSize; + + private final AtomicLong size = new AtomicLong(0); + + private final AtomicBoolean stop = new AtomicBoolean(false); + + private HBaseTestAppender(String name, Layout layout, Filter filter, + boolean ignoreExceptions, boolean immediateFlush, Property[] properties, + OutputStreamManager manager, long maxSize) { + super(name, layout, filter, ignoreExceptions, immediateFlush, properties, manager); + this.maxSize = maxSize; + } + + @Override + public void append(LogEvent event) { + if (stop.get()) { + return; + } + // for accounting, here we always convert the event to byte array first + // this will effect performance a bit but it is OK since this is for UT only + byte[] bytes = getLayout().toByteArray(event); + if (bytes == null || bytes.length == 0) { + return; + } + long sizeAfterAppend = size.addAndGet(bytes.length); + if (sizeAfterAppend >= maxSize) { + // stop logging if the log size exceeded the limit + if (stop.compareAndSet(false, true)) { + LOGGER.error("Log size exceeded the limit {}, will stop logging to prevent eating" + + " too much disk space", maxSize); + } + return; + } + super.append(event); + } +} diff --git a/hbase-logging/src/test/java/org/apache/hadoop/hbase/logging/HBaseTestOutputStreamManager.java b/hbase-logging/src/test/java/org/apache/hadoop/hbase/logging/HBaseTestOutputStreamManager.java new file mode 100644 index 00000000000..4739cbd5e72 --- /dev/null +++ b/hbase-logging/src/test/java/org/apache/hadoop/hbase/logging/HBaseTestOutputStreamManager.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.logging; + +public class HBaseTestOutputStreamManager + extends org.apache.logging.log4j.core.appender.OutputStreamManager { + + public HBaseTestOutputStreamManager(Target target, + org.apache.logging.log4j.core.Layout layout) { + super(target.output(), target.name(), layout, true); + } +} diff --git a/hbase-logging/src/test/java/org/apache/hadoop/hbase/logging/Target.java b/hbase-logging/src/test/java/org/apache/hadoop/hbase/logging/Target.java new file mode 100644 index 00000000000..22b649cd44d --- /dev/null +++ b/hbase-logging/src/test/java/org/apache/hadoop/hbase/logging/Target.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.logging; + +import java.io.PrintStream; + +public enum Target { + SYSTEM_OUT(System.out), + SYSTEM_ERR(System.err); + + private final PrintStream output; + + private Target(PrintStream output) { + this.output = output; + } + + public PrintStream output() { + return output; + } +} \ No newline at end of file diff --git a/hbase-logging/src/test/resources/log4j2.properties b/hbase-logging/src/test/resources/log4j2.properties index f63c8701e35..67c5670effe 100644 --- a/hbase-logging/src/test/resources/log4j2.properties +++ b/hbase-logging/src/test/resources/log4j2.properties @@ -19,10 +19,12 @@ status = debug dest = err name = PropertiesConfig +packages = org.apache.hadoop.hbase.logging -appender.console.type = Console +appender.console.type = HBaseTest appender.console.target = SYSTEM_ERR appender.console.name = Console +appender.console.maxSize = 1G appender.console.layout.type = PatternLayout appender.console.layout.pattern = %d{ISO8601} %-5p [%t] %C{2}(%L): %m%n diff --git a/pom.xml b/pom.xml index b525d977737..9fa5ba61775 100644 --- a/pom.xml +++ b/pom.xml @@ -1781,6 +1781,10 @@ false ${surefire.skipFirstPart} ${surefire.firstPartForkCount} + false ${surefire.reportsDirectory} ${surefire.tempDir} @@ -1825,6 +1829,10 @@ ${surefire.skipSecondPart} ${surefire.testFailureIgnore} + false ${surefire.secondPartForkCount} ${surefire.secondPartGroups} @@ -2331,6 +2339,10 @@ org.apache.log4j.** org.apache.logging.log4j.** + + + org.apache.hadoop.hbase.logging.HBaseTestAppender + false