mirror of https://github.com/apache/druid.git
fix http firehose factory leaky connection in constructor (#8576)
* fix http firehose factory leaky connection in constructor * stylin
This commit is contained in:
parent
7c14fa08f8
commit
eabddffd6e
|
@ -49,7 +49,6 @@ public class HttpFirehoseFactory extends PrefetchableTextFilesFirehoseFactory<UR
|
||||||
{
|
{
|
||||||
private static final Logger log = new Logger(HttpFirehoseFactory.class);
|
private static final Logger log = new Logger(HttpFirehoseFactory.class);
|
||||||
private final List<URI> uris;
|
private final List<URI> uris;
|
||||||
private final boolean supportContentRange;
|
|
||||||
@Nullable
|
@Nullable
|
||||||
private final String httpAuthenticationUsername;
|
private final String httpAuthenticationUsername;
|
||||||
@Nullable
|
@Nullable
|
||||||
|
@ -63,29 +62,25 @@ public class HttpFirehoseFactory extends PrefetchableTextFilesFirehoseFactory<UR
|
||||||
@JsonProperty("prefetchTriggerBytes") Long prefetchTriggerBytes,
|
@JsonProperty("prefetchTriggerBytes") Long prefetchTriggerBytes,
|
||||||
@JsonProperty("fetchTimeout") Long fetchTimeout,
|
@JsonProperty("fetchTimeout") Long fetchTimeout,
|
||||||
@JsonProperty("maxFetchRetry") Integer maxFetchRetry,
|
@JsonProperty("maxFetchRetry") Integer maxFetchRetry,
|
||||||
@Nullable
|
@JsonProperty("httpAuthenticationUsername") @Nullable String httpAuthenticationUsername,
|
||||||
@JsonProperty("httpAuthenticationUsername") String httpAuthenticationUsername,
|
@JsonProperty("httpAuthenticationPassword") @Nullable PasswordProvider httpAuthenticationPasswordProvider
|
||||||
@Nullable
|
)
|
||||||
@JsonProperty("httpAuthenticationPassword") PasswordProvider httpAuthenticationPasswordProvider
|
|
||||||
) throws IOException
|
|
||||||
{
|
{
|
||||||
super(maxCacheCapacityBytes, maxFetchCapacityBytes, prefetchTriggerBytes, fetchTimeout, maxFetchRetry);
|
super(maxCacheCapacityBytes, maxFetchCapacityBytes, prefetchTriggerBytes, fetchTimeout, maxFetchRetry);
|
||||||
this.uris = uris;
|
|
||||||
|
|
||||||
Preconditions.checkArgument(uris.size() > 0, "Empty URIs");
|
Preconditions.checkArgument(uris.size() > 0, "Empty URIs");
|
||||||
final URLConnection connection = uris.get(0).toURL().openConnection();
|
this.uris = uris;
|
||||||
final String acceptRanges = connection.getHeaderField(HttpHeaders.ACCEPT_RANGES);
|
|
||||||
this.supportContentRange = acceptRanges != null && "bytes".equalsIgnoreCase(acceptRanges);
|
|
||||||
this.httpAuthenticationUsername = httpAuthenticationUsername;
|
this.httpAuthenticationUsername = httpAuthenticationUsername;
|
||||||
this.httpAuthenticationPasswordProvider = httpAuthenticationPasswordProvider;
|
this.httpAuthenticationPasswordProvider = httpAuthenticationPasswordProvider;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Nullable
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
public String getHttpAuthenticationUsername()
|
public String getHttpAuthenticationUsername()
|
||||||
{
|
{
|
||||||
return httpAuthenticationUsername;
|
return httpAuthenticationUsername;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Nullable
|
||||||
@JsonProperty("httpAuthenticationPassword")
|
@JsonProperty("httpAuthenticationPassword")
|
||||||
public PasswordProvider getHttpAuthenticationPasswordProvider()
|
public PasswordProvider getHttpAuthenticationPasswordProvider()
|
||||||
{
|
{
|
||||||
|
@ -115,14 +110,16 @@ public class HttpFirehoseFactory extends PrefetchableTextFilesFirehoseFactory<UR
|
||||||
protected InputStream openObjectStream(URI object, long start) throws IOException
|
protected InputStream openObjectStream(URI object, long start) throws IOException
|
||||||
{
|
{
|
||||||
URLConnection urlConnection = openURLConnection(object);
|
URLConnection urlConnection = openURLConnection(object);
|
||||||
if (supportContentRange && start > 0) {
|
final String acceptRanges = urlConnection.getHeaderField(HttpHeaders.ACCEPT_RANGES);
|
||||||
|
final boolean withRanges = "bytes".equalsIgnoreCase(acceptRanges);
|
||||||
|
if (withRanges && start > 0) {
|
||||||
// Set header for range request.
|
// Set header for range request.
|
||||||
// Since we need to set only the start offset, the header is "bytes=<range-start>-".
|
// Since we need to set only the start offset, the header is "bytes=<range-start>-".
|
||||||
// See https://tools.ietf.org/html/rfc7233#section-2.1
|
// See https://tools.ietf.org/html/rfc7233#section-2.1
|
||||||
urlConnection.addRequestProperty(HttpHeaders.RANGE, StringUtils.format("bytes=%d-", start));
|
urlConnection.addRequestProperty(HttpHeaders.RANGE, StringUtils.format("bytes=%d-", start));
|
||||||
return urlConnection.getInputStream();
|
return urlConnection.getInputStream();
|
||||||
} else {
|
} else {
|
||||||
if (!supportContentRange && start > 0) {
|
if (!withRanges && start > 0) {
|
||||||
log.warn(
|
log.warn(
|
||||||
"Since the input source doesn't support range requests, the object input stream is opened from the start and "
|
"Since the input source doesn't support range requests, the object input stream is opened from the start and "
|
||||||
+ "then skipped. This may make the ingestion speed slower. Consider enabling prefetch if you see this message"
|
+ "then skipped. This may make the ingestion speed slower. Consider enabling prefetch if you see this message"
|
||||||
|
@ -159,8 +156,8 @@ public class HttpFirehoseFactory extends PrefetchableTextFilesFirehoseFactory<UR
|
||||||
getPrefetchTriggerBytes() == that.getPrefetchTriggerBytes() &&
|
getPrefetchTriggerBytes() == that.getPrefetchTriggerBytes() &&
|
||||||
getFetchTimeout() == that.getFetchTimeout() &&
|
getFetchTimeout() == that.getFetchTimeout() &&
|
||||||
getMaxFetchRetry() == that.getMaxFetchRetry() &&
|
getMaxFetchRetry() == that.getMaxFetchRetry() &&
|
||||||
httpAuthenticationUsername.equals(that.getHttpAuthenticationUsername()) &&
|
Objects.equals(httpAuthenticationUsername, that.getHttpAuthenticationUsername()) &&
|
||||||
httpAuthenticationPasswordProvider.equals(that.getHttpAuthenticationPasswordProvider());
|
Objects.equals(httpAuthenticationPasswordProvider, that.getHttpAuthenticationPasswordProvider());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -187,21 +184,16 @@ public class HttpFirehoseFactory extends PrefetchableTextFilesFirehoseFactory<UR
|
||||||
@Override
|
@Override
|
||||||
public FiniteFirehoseFactory<StringInputRowParser, URI> withSplit(InputSplit<URI> split)
|
public FiniteFirehoseFactory<StringInputRowParser, URI> withSplit(InputSplit<URI> split)
|
||||||
{
|
{
|
||||||
try {
|
return new HttpFirehoseFactory(
|
||||||
return new HttpFirehoseFactory(
|
Collections.singletonList(split.get()),
|
||||||
Collections.singletonList(split.get()),
|
getMaxCacheCapacityBytes(),
|
||||||
getMaxCacheCapacityBytes(),
|
getMaxFetchCapacityBytes(),
|
||||||
getMaxFetchCapacityBytes(),
|
getPrefetchTriggerBytes(),
|
||||||
getPrefetchTriggerBytes(),
|
getFetchTimeout(),
|
||||||
getFetchTimeout(),
|
getMaxFetchRetry(),
|
||||||
getMaxFetchRetry(),
|
getHttpAuthenticationUsername(),
|
||||||
getHttpAuthenticationUsername(),
|
httpAuthenticationPasswordProvider
|
||||||
httpAuthenticationPasswordProvider
|
);
|
||||||
);
|
|
||||||
}
|
|
||||||
catch (IOException e) {
|
|
||||||
throw new RuntimeException(e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private URLConnection openURLConnection(URI object) throws IOException
|
private URLConnection openURLConnection(URI object) throws IOException
|
||||||
|
|
Loading…
Reference in New Issue