fix tests

This commit is contained in:
Jihoon Son 2019-11-08 23:59:18 -08:00
parent 6dba81a046
commit ea2c8f9db6
4 changed files with 16 additions and 24 deletions

View File

@ -34,7 +34,6 @@ import org.apache.druid.data.input.SplitHintSpec;
import javax.annotation.Nullable;
import java.io.File;
import java.io.FileNotFoundException;
import java.util.Iterator;
import java.util.Objects;
import java.util.Spliterator;
@ -117,14 +116,7 @@ public class LocalInputSource extends AbstractInputSource implements SplittableI
// reader() is supposed to be called in each task that creates segments.
// The task should already have only one split in parallel indexing,
// while there's no need to make splits using splitHintSpec in sequential indexing.
createSplits(inputFormat, null).map(split -> {
try {
return new FileSource(split.get());
}
catch (FileNotFoundException e) {
throw new RuntimeException(e);
}
}),
createSplits(inputFormat, null).map(split -> new FileSource(split.get())),
temporaryDirectory
);
}

View File

@ -287,16 +287,23 @@ public class FileUtils
String messageOnRetry
) throws IOException
{
try (InputStream inputStream = objectOpenFunction.open(object)) {
return copyLarge(
inputStream,
outFile,
fetchBuffer,
try {
return RetryUtils.retry(
() -> {
try (InputStream inputStream = objectOpenFunction.open(object);
OutputStream out = new FileOutputStream(outFile)) {
return IOUtils.copyLarge(inputStream, out, fetchBuffer);
}
},
retryCondition,
outFile::delete,
numRetries,
messageOnRetry
);
}
catch (Exception e) {
throw new IOException(e);
}
}
public static long copyLarge(

View File

@ -72,14 +72,7 @@ public class ObjectIteratingReaderTest
false,
0
),
files.stream().flatMap(file -> {
try {
return ImmutableList.of(new FileSource(file)).stream();
}
catch (IOException e) {
throw new RuntimeException(e);
}
}),
files.stream().flatMap(file -> ImmutableList.of(new FileSource(file)).stream()),
temporaryFolder.newFolder()
);

View File

@ -198,7 +198,7 @@ public class DataSchema
}
@Nullable
@JsonProperty
@JsonProperty("timestampSpec")
private TimestampSpec getGivenTimestampSpec()
{
return timestampSpec;
@ -213,7 +213,7 @@ public class DataSchema
}
@Nullable
@JsonProperty
@JsonProperty("dimensionsSpec")
private DimensionsSpec getGivenDimensionsSpec()
{
return dimensionsSpec;