feature/BAEL-6712-split large file (#15262)
* feature/BAEL-6712-split lareg file * feature/BAEL-6712-split lareg file * bael-6712 review comment fix --------- Co-authored-by: Sachin kumar <sachin.n.kumar@oracle.com>
This commit is contained in:
parent
af56780cfa
commit
b19a80d569
@ -0,0 +1,59 @@
|
||||
package com.baeldung.splitlargefile;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
|
||||
class SplitLargeFile {
|
||||
|
||||
public List<File> splitByFileSize(File largeFile, int maxSizeOfSplitFiles,
|
||||
String splitedFileDirPath) throws IOException {
|
||||
List<File> listOfSplitFiles = new ArrayList<>();
|
||||
try (InputStream in = Files.newInputStream(largeFile.toPath())) {
|
||||
final byte[] buffer = new byte[maxSizeOfSplitFiles];
|
||||
int dataRead = in.read(buffer);
|
||||
while (dataRead > -1) {
|
||||
File splitFile = getSplitFile(FilenameUtils.removeExtension(largeFile.getName()),
|
||||
buffer, dataRead, splitedFileDirPath);
|
||||
listOfSplitFiles.add(splitFile);
|
||||
dataRead = in.read(buffer);
|
||||
}
|
||||
}
|
||||
return listOfSplitFiles;
|
||||
}
|
||||
|
||||
private File getSplitFile(String largeFileName, byte[] buffer, int length,
|
||||
String splitedFileDirPath) throws IOException {
|
||||
File splitFile = File.createTempFile(largeFileName + "-", "-split",
|
||||
new File(splitedFileDirPath));
|
||||
try (FileOutputStream fos = new FileOutputStream(splitFile)) {
|
||||
fos.write(buffer, 0, length);
|
||||
}
|
||||
return splitFile;
|
||||
}
|
||||
|
||||
public List<File> splitByNumberOfFiles(File largeFile, int noOfFiles, String splitedFileDirPath)
|
||||
throws IOException {
|
||||
return splitByFileSize(largeFile, getSizeInBytes(largeFile.length(), noOfFiles),
|
||||
splitedFileDirPath);
|
||||
}
|
||||
|
||||
private int getSizeInBytes(long largefileSizeInBytes, int numberOfFilesforSplit) {
|
||||
if (largefileSizeInBytes % numberOfFilesforSplit != 0) {
|
||||
largefileSizeInBytes = ((largefileSizeInBytes / numberOfFilesforSplit) + 1)
|
||||
* numberOfFilesforSplit;
|
||||
}
|
||||
long x = largefileSizeInBytes / numberOfFilesforSplit;
|
||||
if (x > Integer.MAX_VALUE) {
|
||||
throw new NumberFormatException("size too large");
|
||||
|
||||
}
|
||||
return (int) x;
|
||||
}
|
||||
}
|
@ -0,0 +1,41 @@
|
||||
package com.baeldung.splitlargefile;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
|
||||
public class SplitLargeFileUnitTest {
|
||||
|
||||
@BeforeClass
|
||||
public static void prepareData() throws IOException {
|
||||
Files.createDirectories(Paths.get("target/split"));
|
||||
}
|
||||
|
||||
private String splitedFileDirPath() throws Exception {
|
||||
return Paths.get("target").toString() + "/split";
|
||||
}
|
||||
|
||||
private Path largeFilePath() throws Exception {
|
||||
return Paths.get(this.getClass().getClassLoader().getResource("large-file.txt").toURI());
|
||||
}
|
||||
|
||||
|
||||
@org.junit.Test
|
||||
public void givenLargeFile_whenSplitLargeFile_thenSplitBySize() throws Exception {
|
||||
File input = largeFilePath().toFile();
|
||||
SplitLargeFile slf = new SplitLargeFile();
|
||||
slf.splitByFileSize(input, 1024_000, splitedFileDirPath());
|
||||
}
|
||||
|
||||
@org.junit.Test
|
||||
public void givenLargeFile_whenSplitLargeFile_thenSplitByNumberOfFiles() throws Exception {
|
||||
File input = largeFilePath().toFile();
|
||||
SplitLargeFile slf = new SplitLargeFile();
|
||||
slf.splitByNumberOfFiles(input, 3, splitedFileDirPath());
|
||||
}
|
||||
|
||||
}
|
Loading…
x
Reference in New Issue
Block a user