Copy closer into Druid codebase (fixes #3652) (#4153)

This commit is contained in:
Roman Leventov 2017-04-09 17:38:45 -07:00 committed by Gian Merlino
parent 7e0d4c9555
commit 15f3a94474
26 changed files with 266 additions and 25 deletions

8
NOTICE
View File

@ -69,3 +69,11 @@ ConciseSet to use IntBuffers.
* Apache License, Version 2.0
* HOMEPAGE:
* https://sourceforge.net/projects/concise/
This product contains a modified version of The Guava Authors's Closer class from Guava library:
* LICENSE:
* https://github.com/google/guava/blob/c462d69329709f72a17a64cb229d15e76e72199c/COPYING (Apache License, Version 2.0)
* HOMEPAGE:
* https://github.com/google/guava
* COMMIT TAG:
* https://github.com/google/guava/blob/c462d69329709f72a17a64cb229d15e76e72199c

View File

@ -43,5 +43,10 @@
<module name="RedundantImport"/>
<module name="UnusedImports" />
<module name="NeedBraces"/>
<module name="Regexp">
<!-- Prohibit using Guava's Closer, see comment in io.druid.java.util.common.io.Closer -->
<property name="format" value="com\.google\.common\.io\.Closer"/>
<property name="illegalPattern" value="true"/>
</module>
</module>
</module>

View File

@ -22,8 +22,7 @@ package io.druid.collections;
import com.google.common.base.Function;
import com.google.common.base.Throwables;
import com.google.common.collect.Ordering;
import com.google.common.io.Closer;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.guava.Accumulator;
import io.druid.java.util.common.guava.CloseQuietly;
import io.druid.java.util.common.guava.Sequence;

View File

@ -22,16 +22,15 @@ package io.druid.query.lookup;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.io.Closer;
import com.google.inject.Binder;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.name.Names;
import io.druid.guice.GuiceInjectors;
import io.druid.initialization.Initialization;
import io.druid.java.util.common.ISE;
import io.druid.java.util.common.StringUtils;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.logger.Logger;
import io.druid.server.lookup.namespace.NamespaceExtractionModule;
import kafka.admin.AdminUtils;

View File

@ -23,12 +23,11 @@ import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.Closer;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import io.druid.concurrent.Execs;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.lifecycle.Lifecycle;
import io.druid.java.util.common.logger.Logger;
import io.druid.metadata.TestDerbyConnector;

View File

@ -36,7 +36,6 @@ import com.google.common.collect.Maps;
import com.google.common.io.ByteSink;
import com.google.common.io.ByteSource;
import com.google.common.io.ByteStreams;
import com.google.common.io.Closer;
import com.google.common.io.FileWriteMode;
import com.google.common.io.Files;
import com.google.common.util.concurrent.ListenableFuture;
@ -54,6 +53,7 @@ import io.druid.indexing.common.tasklogs.LogUtils;
import io.druid.indexing.overlord.autoscaling.ScalingStats;
import io.druid.indexing.overlord.config.ForkingTaskRunnerConfig;
import io.druid.indexing.worker.config.WorkerConfig;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.ISE;
import io.druid.java.util.common.Pair;
import io.druid.java.util.common.lifecycle.LifecycleStop;

View File

@ -37,7 +37,6 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.io.ByteSource;
import com.google.common.io.Closer;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
@ -64,6 +63,7 @@ import io.druid.indexing.overlord.setup.WorkerBehaviorConfig;
import io.druid.indexing.overlord.setup.WorkerSelectStrategy;
import io.druid.indexing.worker.TaskAnnouncement;
import io.druid.indexing.worker.Worker;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.ISE;
import io.druid.java.util.common.Pair;
import io.druid.java.util.common.RE;

View File

@ -85,6 +85,10 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
<!-- Tests -->
<dependency>

View File

@ -22,7 +22,7 @@ package io.druid.java.util.common.guava;
import com.google.common.base.Function;
import com.google.common.base.Throwables;
import com.google.common.collect.Ordering;
import com.google.common.io.Closer;
import io.druid.java.util.common.io.Closer;
import java.io.IOException;
import java.util.PriorityQueue;

View File

@ -0,0 +1,227 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.java.util.common.io;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import javax.annotation.Nullable;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.Deque;
/**
* A {@link Closeable} that collects {@code Closeable} resources and closes them all when it is
* {@linkplain #close closed}. This is intended to approximately emulate the behavior of Java 7's
* <a href="http://docs.oracle.com/javase/tutorial/essential/exceptions/tryResourceClose.html"
* >try-with-resources</a> statement in JDK6-compatible code. Running on Java 7, code using this
* should be approximately equivalent in behavior to the same code written with try-with-resources.
* Running on Java 6, exceptions that cannot be thrown must be logged rather than being added to the
* thrown exception as a suppressed exception.
*
* <p>This class is intended to be used in the following pattern:
*
* <pre> {@code
* Closer closer = Closer.create();
* try {
* InputStream in = closer.register(openInputStream());
* OutputStream out = closer.register(openOutputStream());
* // do stuff
* } catch (Throwable e) {
* // ensure that any checked exception types other than IOException that could be thrown are
* // provided here, e.g. throw closer.rethrow(e, CheckedException.class);
* throw closer.rethrow(e);
* } finally {
* closer.close();
* }}</pre>
*
* <p>Note that this try-catch-finally block is not equivalent to a try-catch-finally block using
* try-with-resources. To get the equivalent of that, you must wrap the above code in <i>another</i>
* try block in order to catch any exception that may be thrown (including from the call to
* {@code close()}).
*
* <p>This pattern ensures the following:
*
* <ul>
* <li>Each {@code Closeable} resource that is successfully registered will be closed later.
* <li>If a {@code Throwable} is thrown in the try block, no exceptions that occur when attempting
* to close resources will be thrown from the finally block. The throwable from the try block
* will be thrown.
* <li>If no exceptions or errors were thrown in the try block, the <i>first</i> exception thrown by
* an attempt to close a resource will be thrown.
* <li>Any exception caught when attempting to close a resource that is <i>not</i> thrown (because
* another exception is already being thrown) is <i>suppressed</i>.
* </ul>
*
* <p>An exception that is suppressed is not thrown. The method of suppression used depends on the
* version of Java the code is running on:
*
* <ul>
* <li><b>Java 7+:</b> Exceptions are suppressed by adding them to the exception that <i>will</i> be
* thrown using {@code Throwable.addSuppressed(Throwable)}.
* <li><b>Java 6:</b> Exceptions are suppressed by logging them instead.
* </ul>
*
* ===========================
* <p>Closer is introduced in Guava 14.0, and not present in Guava 11.0 on which Hadoop depends. We had to be very
* careful to ensure that Closer is not used in code that might be executed by Hadoop indexing service. We copy Closer
* to Druid codebase in order to remove this headache.
*
* @author Colin Decker
* @since Guava 14.0
*/
// Coffee's for {@link Closer closers} only.
public final class Closer implements Closeable {
/**
* Creates a new {@link Closer}.
*/
public static Closer create() {
return new Closer();
}
// only need space for 2 elements in most cases, so try to use the smallest array possible
private final Deque<Closeable> stack = new ArrayDeque<>(4);
private Throwable thrown;
private Closer()
{
}
/**
* Registers the given {@code closeable} to be closed when this {@code Closer} is
* {@linkplain #close closed}.
*
* @return the given {@code closeable}
*/
// close. this word no longer has any meaning to me.
public <C extends Closeable> C register(@Nullable C closeable) {
if (closeable != null) {
stack.addFirst(closeable);
}
return closeable;
}
/**
* Stores the given throwable and rethrows it. It will be rethrown as is if it is an
* {@code IOException}, {@code RuntimeException} or {@code Error}. Otherwise, it will be rethrown
* wrapped in a {@code RuntimeException}. <b>Note:</b> Be sure to declare all of the checked
* exception types your try block can throw when calling an overload of this method so as to avoid
* losing the original exception type.
*
* <p>This method always throws, and as such should be called as {@code throw closer.rethrow(e);}
* to ensure the compiler knows that it will throw.
*
* @return this method does not return; it always throws
* @throws IOException when the given throwable is an IOException
*/
public RuntimeException rethrow(Throwable e) throws IOException {
Preconditions.checkNotNull(e);
thrown = e;
Throwables.propagateIfPossible(e, IOException.class);
throw new RuntimeException(e);
}
/**
* Stores the given throwable and rethrows it. It will be rethrown as is if it is an
* {@code IOException}, {@code RuntimeException}, {@code Error} or a checked exception of the
* given type. Otherwise, it will be rethrown wrapped in a {@code RuntimeException}. <b>Note:</b>
* Be sure to declare all of the checked exception types your try block can throw when calling an
* overload of this method so as to avoid losing the original exception type.
*
* <p>This method always throws, and as such should be called as
* {@code throw closer.rethrow(e, ...);} to ensure the compiler knows that it will throw.
*
* @return this method does not return; it always throws
* @throws IOException when the given throwable is an IOException
* @throws X when the given throwable is of the declared type X
*/
public <X extends Exception> RuntimeException rethrow(Throwable e, Class<X> declaredType)
throws IOException, X {
Preconditions.checkNotNull(e);
thrown = e;
Throwables.propagateIfPossible(e, IOException.class);
Throwables.propagateIfPossible(e, declaredType);
throw new RuntimeException(e);
}
/**
* Stores the given throwable and rethrows it. It will be rethrown as is if it is an
* {@code IOException}, {@code RuntimeException}, {@code Error} or a checked exception of either
* of the given types. Otherwise, it will be rethrown wrapped in a {@code RuntimeException}.
* <b>Note:</b> Be sure to declare all of the checked exception types your try block can throw
* when calling an overload of this method so as to avoid losing the original exception type.
*
* <p>This method always throws, and as such should be called as
* {@code throw closer.rethrow(e, ...);} to ensure the compiler knows that it will throw.
*
* @return this method does not return; it always throws
* @throws IOException when the given throwable is an IOException
* @throws X1 when the given throwable is of the declared type X1
* @throws X2 when the given throwable is of the declared type X2
*/
public <X1 extends Exception, X2 extends Exception> RuntimeException rethrow(
Throwable e, Class<X1> declaredType1, Class<X2> declaredType2) throws IOException, X1, X2 {
Preconditions.checkNotNull(e);
thrown = e;
Throwables.propagateIfPossible(e, IOException.class);
Throwables.propagateIfPossible(e, declaredType1, declaredType2);
throw new RuntimeException(e);
}
/**
* Closes all {@code Closeable} instances that have been added to this {@code Closer}. If an
* exception was thrown in the try block and passed to one of the {@code exceptionThrown} methods,
* any exceptions thrown when attempting to close a closeable will be suppressed. Otherwise, the
* <i>first</i> exception to be thrown from an attempt to close a closeable will be thrown and any
* additional exceptions that are thrown after that will be suppressed.
*/
@Override
public void close() throws IOException {
Throwable throwable = thrown;
// close closeables in LIFO order
while (!stack.isEmpty()) {
Closeable closeable = stack.removeFirst();
try {
closeable.close();
} catch (Throwable e) {
if (throwable == null) {
throwable = e;
} else {
suppress(throwable, e);
}
}
}
if (thrown == null && throwable != null) {
Throwables.propagateIfPossible(throwable, IOException.class);
throw new AssertionError(throwable); // not possible
}
}
private void suppress(Throwable thrown, Throwable suppressed) {
if(thrown != suppressed) {
thrown.addSuppressed(suppressed);
}
}
}

View File

@ -25,12 +25,12 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.io.ByteStreams;
import com.google.common.io.Closer;
import com.google.common.primitives.Ints;
import io.druid.java.util.common.FileUtils;
import io.druid.java.util.common.IAE;
import io.druid.java.util.common.ISE;
import io.druid.java.util.common.MappedByteBufferHandler;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.logger.Logger;
import java.io.BufferedWriter;

View File

@ -19,7 +19,7 @@
package io.druid.segment;
import com.google.common.io.Closer;
import io.druid.java.util.common.io.Closer;
import java.io.IOException;
import java.nio.IntBuffer;

View File

@ -20,7 +20,7 @@
package io.druid.segment;
import com.google.common.base.Throwables;
import com.google.common.io.Closer;
import io.druid.java.util.common.io.Closer;
import io.druid.segment.column.ColumnCapabilities;
import io.druid.segment.column.ColumnDescriptor;
import io.druid.segment.column.ValueType;

View File

@ -32,7 +32,6 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.io.ByteStreams;
import com.google.common.io.Closeables;
import com.google.common.io.Closer;
import com.google.common.io.Files;
import com.google.common.primitives.Ints;
import com.google.inject.Inject;
@ -43,6 +42,7 @@ import io.druid.collections.bitmap.ImmutableBitmap;
import io.druid.collections.bitmap.MutableBitmap;
import io.druid.collections.spatial.ImmutableRTree;
import io.druid.common.utils.SerializerUtils;
import io.druid.java.util.common.io.Closer;
import io.druid.io.ZeroCopyByteArrayOutputStream;
import io.druid.java.util.common.IAE;
import io.druid.java.util.common.ISE;

View File

@ -33,7 +33,6 @@ import com.google.common.collect.Ordering;
import com.google.common.collect.PeekingIterator;
import com.google.common.collect.Sets;
import com.google.common.io.ByteSink;
import com.google.common.io.Closer;
import com.google.common.io.FileWriteMode;
import com.google.common.io.Files;
import com.google.common.io.OutputSupplier;
@ -45,6 +44,7 @@ import io.druid.common.guava.FileOutputSupplier;
import io.druid.common.guava.GuavaUtils;
import io.druid.common.utils.JodaUtils;
import io.druid.common.utils.SerializerUtils;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.IAE;
import io.druid.java.util.common.ISE;
import io.druid.java.util.common.Pair;

View File

@ -26,11 +26,11 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Ordering;
import com.google.common.collect.Sets;
import com.google.common.io.ByteStreams;
import com.google.common.io.Closer;
import com.google.common.io.Files;
import com.google.common.primitives.Ints;
import com.google.inject.Inject;
import io.druid.common.utils.JodaUtils;
import io.druid.java.util.common.io.Closer;
import io.druid.io.ZeroCopyByteArrayOutputStream;
import io.druid.java.util.common.ISE;
import io.druid.java.util.common.io.smoosh.FileSmoosher;

View File

@ -20,7 +20,7 @@
package io.druid.segment;
import com.google.common.base.Throwables;
import com.google.common.io.Closer;
import io.druid.java.util.common.io.Closer;
import io.druid.segment.column.ColumnCapabilities;
import io.druid.segment.column.ColumnDescriptor;
import io.druid.segment.column.ValueType;

View File

@ -24,7 +24,7 @@ import com.google.common.base.Function;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Closer;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.ISE;
import io.druid.java.util.common.guava.CloseQuietly;
import io.druid.java.util.common.logger.Logger;

View File

@ -26,8 +26,8 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.io.Closer;
import io.druid.collections.bitmap.ImmutableBitmap;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.granularity.Granularity;
import io.druid.java.util.common.guava.Sequence;
import io.druid.java.util.common.guava.Sequences;

View File

@ -21,7 +21,6 @@ package io.druid.segment;
import com.google.common.io.ByteSink;
import com.google.common.io.ByteStreams;
import com.google.common.io.Closer;
import com.google.common.io.Files;
import com.google.common.io.OutputSupplier;
import com.google.common.primitives.Ints;
@ -31,6 +30,7 @@ import io.druid.collections.spatial.RTree;
import io.druid.collections.spatial.split.LinearGutmanSplitStrategy;
import io.druid.common.guava.FileOutputSupplier;
import io.druid.common.utils.SerializerUtils;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.ByteBufferUtils;
import io.druid.java.util.common.ISE;
import io.druid.java.util.common.logger.Logger;

View File

@ -23,7 +23,6 @@ import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.io.ByteStreams;
import com.google.common.io.Closer;
import com.google.common.io.Files;
import io.druid.collections.bitmap.BitmapFactory;
import io.druid.collections.bitmap.ImmutableBitmap;
@ -31,6 +30,7 @@ import io.druid.collections.bitmap.MutableBitmap;
import io.druid.collections.spatial.ImmutableRTree;
import io.druid.collections.spatial.RTree;
import io.druid.collections.spatial.split.LinearGutmanSplitStrategy;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.ByteBufferUtils;
import io.druid.java.util.common.ISE;
import io.druid.java.util.common.logger.Logger;

View File

@ -22,9 +22,9 @@ package io.druid.segment.incremental;
import com.google.common.base.Supplier;
import com.google.common.base.Throwables;
import com.google.common.collect.Maps;
import com.google.common.io.Closer;
import io.druid.data.input.InputRow;
import io.druid.data.input.impl.DimensionsSpec;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.granularity.Granularity;
import io.druid.java.util.common.logger.Logger;
import io.druid.java.util.common.parsers.ParseException;

View File

@ -19,7 +19,7 @@
package io.druid.segment;
import com.google.common.io.Closer;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.logger.Logger;
import org.junit.rules.TestRule;
import org.junit.runner.Description;

View File

@ -23,8 +23,8 @@ import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker;
import com.google.common.collect.Sets;
import com.google.common.io.Closer;
import io.druid.curator.cache.PathChildrenCacheFactory;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.IAE;
import io.druid.java.util.common.ISE;
import io.druid.java.util.common.guava.CloseQuietly;

View File

@ -23,8 +23,8 @@ import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import com.google.common.collect.MapMaker;
import com.google.common.collect.Sets;
import com.google.common.io.Closer;
import io.druid.curator.cache.PathChildrenCacheFactory;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.lifecycle.LifecycleStart;
import io.druid.java.util.common.lifecycle.LifecycleStop;
import io.druid.java.util.common.logger.Logger;

View File

@ -26,9 +26,9 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.io.Closer;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.inject.Inject;
import io.druid.java.util.common.io.Closer;
import io.druid.java.util.common.ISE;
import io.druid.java.util.common.logger.Logger;
import io.druid.sql.calcite.planner.Calcites;