feature/BAEL-6712-split large file (#15262)

* feature/BAEL-6712-split lareg file

* feature/BAEL-6712-split lareg file

* bael-6712 review comment fix

---------

Co-authored-by: Sachin kumar <sachin.n.kumar@oracle.com>
This commit is contained in:
sachin 2023-12-20 07:04:54 +05:30 committed by GitHub
parent af56780cfa
commit b19a80d569
2 changed files with 100 additions and 0 deletions

View File

@ -0,0 +1,59 @@
package com.baeldung.splitlargefile;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.io.FilenameUtils;
class SplitLargeFile {
public List<File> splitByFileSize(File largeFile, int maxSizeOfSplitFiles,
String splitedFileDirPath) throws IOException {
List<File> listOfSplitFiles = new ArrayList<>();
try (InputStream in = Files.newInputStream(largeFile.toPath())) {
final byte[] buffer = new byte[maxSizeOfSplitFiles];
int dataRead = in.read(buffer);
while (dataRead > -1) {
File splitFile = getSplitFile(FilenameUtils.removeExtension(largeFile.getName()),
buffer, dataRead, splitedFileDirPath);
listOfSplitFiles.add(splitFile);
dataRead = in.read(buffer);
}
}
return listOfSplitFiles;
}
private File getSplitFile(String largeFileName, byte[] buffer, int length,
String splitedFileDirPath) throws IOException {
File splitFile = File.createTempFile(largeFileName + "-", "-split",
new File(splitedFileDirPath));
try (FileOutputStream fos = new FileOutputStream(splitFile)) {
fos.write(buffer, 0, length);
}
return splitFile;
}
public List<File> splitByNumberOfFiles(File largeFile, int noOfFiles, String splitedFileDirPath)
throws IOException {
return splitByFileSize(largeFile, getSizeInBytes(largeFile.length(), noOfFiles),
splitedFileDirPath);
}
private int getSizeInBytes(long largefileSizeInBytes, int numberOfFilesforSplit) {
if (largefileSizeInBytes % numberOfFilesforSplit != 0) {
largefileSizeInBytes = ((largefileSizeInBytes / numberOfFilesforSplit) + 1)
* numberOfFilesforSplit;
}
long x = largefileSizeInBytes / numberOfFilesforSplit;
if (x > Integer.MAX_VALUE) {
throw new NumberFormatException("size too large");
}
return (int) x;
}
}

View File

@ -0,0 +1,41 @@
package com.baeldung.splitlargefile;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.junit.BeforeClass;
public class SplitLargeFileUnitTest {
@BeforeClass
public static void prepareData() throws IOException {
Files.createDirectories(Paths.get("target/split"));
}
private String splitedFileDirPath() throws Exception {
return Paths.get("target").toString() + "/split";
}
private Path largeFilePath() throws Exception {
return Paths.get(this.getClass().getClassLoader().getResource("large-file.txt").toURI());
}
@org.junit.Test
public void givenLargeFile_whenSplitLargeFile_thenSplitBySize() throws Exception {
File input = largeFilePath().toFile();
SplitLargeFile slf = new SplitLargeFile();
slf.splitByFileSize(input, 1024_000, splitedFileDirPath());
}
@org.junit.Test
public void givenLargeFile_whenSplitLargeFile_thenSplitByNumberOfFiles() throws Exception {
File input = largeFilePath().toFile();
SplitLargeFile slf = new SplitLargeFile();
slf.splitByNumberOfFiles(input, 3, splitedFileDirPath());
}
}