feature/BAEL-6712-split large file (#15262)
* feature/BAEL-6712-split lareg file * feature/BAEL-6712-split lareg file * bael-6712 review comment fix --------- Co-authored-by: Sachin kumar <sachin.n.kumar@oracle.com>
This commit is contained in:
parent
af56780cfa
commit
b19a80d569
|
@ -0,0 +1,59 @@
|
||||||
|
package com.baeldung.splitlargefile;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.commons.io.FilenameUtils;
|
||||||
|
|
||||||
|
class SplitLargeFile {
|
||||||
|
|
||||||
|
public List<File> splitByFileSize(File largeFile, int maxSizeOfSplitFiles,
|
||||||
|
String splitedFileDirPath) throws IOException {
|
||||||
|
List<File> listOfSplitFiles = new ArrayList<>();
|
||||||
|
try (InputStream in = Files.newInputStream(largeFile.toPath())) {
|
||||||
|
final byte[] buffer = new byte[maxSizeOfSplitFiles];
|
||||||
|
int dataRead = in.read(buffer);
|
||||||
|
while (dataRead > -1) {
|
||||||
|
File splitFile = getSplitFile(FilenameUtils.removeExtension(largeFile.getName()),
|
||||||
|
buffer, dataRead, splitedFileDirPath);
|
||||||
|
listOfSplitFiles.add(splitFile);
|
||||||
|
dataRead = in.read(buffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return listOfSplitFiles;
|
||||||
|
}
|
||||||
|
|
||||||
|
private File getSplitFile(String largeFileName, byte[] buffer, int length,
|
||||||
|
String splitedFileDirPath) throws IOException {
|
||||||
|
File splitFile = File.createTempFile(largeFileName + "-", "-split",
|
||||||
|
new File(splitedFileDirPath));
|
||||||
|
try (FileOutputStream fos = new FileOutputStream(splitFile)) {
|
||||||
|
fos.write(buffer, 0, length);
|
||||||
|
}
|
||||||
|
return splitFile;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<File> splitByNumberOfFiles(File largeFile, int noOfFiles, String splitedFileDirPath)
|
||||||
|
throws IOException {
|
||||||
|
return splitByFileSize(largeFile, getSizeInBytes(largeFile.length(), noOfFiles),
|
||||||
|
splitedFileDirPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
private int getSizeInBytes(long largefileSizeInBytes, int numberOfFilesforSplit) {
|
||||||
|
if (largefileSizeInBytes % numberOfFilesforSplit != 0) {
|
||||||
|
largefileSizeInBytes = ((largefileSizeInBytes / numberOfFilesforSplit) + 1)
|
||||||
|
* numberOfFilesforSplit;
|
||||||
|
}
|
||||||
|
long x = largefileSizeInBytes / numberOfFilesforSplit;
|
||||||
|
if (x > Integer.MAX_VALUE) {
|
||||||
|
throw new NumberFormatException("size too large");
|
||||||
|
|
||||||
|
}
|
||||||
|
return (int) x;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,41 @@
|
||||||
|
package com.baeldung.splitlargefile;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
public class SplitLargeFileUnitTest {
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void prepareData() throws IOException {
|
||||||
|
Files.createDirectories(Paths.get("target/split"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String splitedFileDirPath() throws Exception {
|
||||||
|
return Paths.get("target").toString() + "/split";
|
||||||
|
}
|
||||||
|
|
||||||
|
private Path largeFilePath() throws Exception {
|
||||||
|
return Paths.get(this.getClass().getClassLoader().getResource("large-file.txt").toURI());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@org.junit.Test
|
||||||
|
public void givenLargeFile_whenSplitLargeFile_thenSplitBySize() throws Exception {
|
||||||
|
File input = largeFilePath().toFile();
|
||||||
|
SplitLargeFile slf = new SplitLargeFile();
|
||||||
|
slf.splitByFileSize(input, 1024_000, splitedFileDirPath());
|
||||||
|
}
|
||||||
|
|
||||||
|
@org.junit.Test
|
||||||
|
public void givenLargeFile_whenSplitLargeFile_thenSplitByNumberOfFiles() throws Exception {
|
||||||
|
File input = largeFilePath().toFile();
|
||||||
|
SplitLargeFile slf = new SplitLargeFile();
|
||||||
|
slf.splitByNumberOfFiles(input, 3, splitedFileDirPath());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in New Issue