Apache
/
poi
mirror of https://github.com/apache/poi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[bug-65613] experimental SXSSF writeAvoidingTempFiles 

git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1893896 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
PJ Fanning 2021-10-04 20:34:47 +00:00
parent 87d4716880
commit 51ce81924d
3 changed files with 126 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
poi-ooxml/src/main/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java
View File

@ -17,13 +17,7 @@
package org.apache.poi.xssf.streaming;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.Enumeration;
import java.util.HashMap;
@ -35,13 +29,16 @@ import java.util.NoSuchElementException;
import org.apache.commons.compress.archivers.ArchiveOutputStream;
import org.apache.commons.compress.archivers.zip.Zip64Mode;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.openxml4j.util.ZipArchiveThresholdInputStream;
import org.apache.poi.openxml4j.util.ZipEntrySource;
import org.apache.poi.openxml4j.util.ZipFileZipEntrySource;
import org.apache.poi.openxml4j.util.ZipInputStreamZipEntrySource;
import org.apache.poi.openxml4j.util.ZipSecureFile;
import org.apache.poi.ss.SpreadsheetVersion;
import org.apache.poi.ss.formula.EvaluationWorkbook;
@ -403,8 +400,8 @@ public class SXSSFWorkbook implements Workbook {
while (en.hasMoreElements()) {
ZipArchiveEntry ze = en.nextElement();
ZipArchiveEntry zeOut = new ZipArchiveEntry(ze.getName());
zeOut.setSize(ze.getSize());
zeOut.setTime(ze.getTime());
if (ze.getSize() >= 0) zeOut.setSize(ze.getSize());
if (ze.getTime() >= 0) zeOut.setTime(ze.getTime());
zos.putArchiveEntry(zeOut);
try (final InputStream is = zipEntrySource.getInputStream(ze)) {
if (is instanceof ZipArchiveThresholdInputStream) {
@ -968,11 +965,38 @@ public class SXSSFWorkbook implements Workbook {
} finally {
deleted = tmplFile.delete();
}
if(!deleted) {
if (!deleted) {
throw new IOException("Could not delete temporary file after processing: " + tmplFile);
}
}
/**
* Write out this workbook to an OutputStream. This (experimental) method avoids the temp file that
* {@link #write} creates but will use more memory as a result. Other SXSSF code can create temp files,
* so using this does not guarantee that there will be no temp file usage.
*
* @param stream - the java OutputStream you wish to write to
* @exception IOException if anything can't be written.
*/
@Beta
public void writeAvoidingTempFiles(OutputStream stream) throws IOException {
flushSheets();
//Save the template
try (UnsynchronizedByteArrayOutputStream bos = new UnsynchronizedByteArrayOutputStream()) {
_wb.write(bos);
//Substitute the template entries with the generated sheet data files
try (
InputStream is = bos.toInputStream();
ZipInputStreamZipEntrySource source = new ZipInputStreamZipEntrySource(
new ZipArchiveThresholdInputStream(new ZipArchiveInputStream(is)))
) {
injectData(source, stream);
}
}
}
protected void flushSheets() throws IOException {
for (SXSSFSheet sheet : _xFromSxHash.values())
{

14
poi-ooxml/src/test/java/org/apache/poi/xssf/DeferredSXSSFITestDataProvider.java
View File

@ -78,6 +78,20 @@ public final class DeferredSXSSFITestDataProvider implements ITestDataProvider {
}
}
/**
* Returns an XSSFWorkbook since SXSSFWorkbook is write-only
*/
public XSSFWorkbook inMemoryWriteOutAndReadBack(SXSSFWorkbook wb) {
try (UnsynchronizedByteArrayOutputStream baos = new UnsynchronizedByteArrayOutputStream()) {
wb.writeAvoidingTempFiles(baos);
try (InputStream is = baos.toInputStream()) {
return new XSSFWorkbook(is);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public DeferredSXSSFWorkbook createWorkbook() {
DeferredSXSSFWorkbook wb = new DeferredSXSSFWorkbook();

78
poi-ooxml/src/test/java/org/apache/poi/xssf/streaming/TestDeferredSXSSFWorkbook.java
View File

@ -187,6 +187,84 @@ public final class TestDeferredSXSSFWorkbook extends BaseTestXWorkbook {
}
}
@Test
void inMemoryWrite() throws IOException {
try (XSSFWorkbook xssfWb1 = new XSSFWorkbook()) {
xssfWb1.createSheet("S1");
Sheet sheet = xssfWb1.createSheet("S2");
Row row = sheet.createRow(1);
Cell cell = row.createCell(1);
cell.setCellValue("value 2_1_1");
try (DeferredSXSSFWorkbook wb1 = new DeferredSXSSFWorkbook(xssfWb1);
XSSFWorkbook xssfWb2 = DeferredSXSSFITestDataProvider.instance.writeOutAndReadBack(wb1)) {
assertTrue(wb1.dispose());
try (DeferredSXSSFWorkbook wb2 = new DeferredSXSSFWorkbook(xssfWb2)) {
// Add a row to the existing empty sheet
DeferredSXSSFSheet ssheet1 = wb2.getStreamingSheetAt(0);
ssheet1.setRowGenerator((ssxSheet) -> {
Row row1_1 = ssxSheet.createRow(1);
Cell cell1_1_1 = row1_1.createCell(1);
cell1_1_1.setCellValue("value 1_1_1");
});
// Add a row to the existing non-empty sheet
DeferredSXSSFSheet ssheet2 = wb2.getStreamingSheetAt(1);
ssheet2.setRowGenerator((ssxSheet) -> {
Row row2_2 = ssxSheet.createRow(2);
Cell cell2_2_1 = row2_2.createCell(1);
cell2_2_1.setCellValue("value 2_2_1");
});
// Add a sheet with one row
DeferredSXSSFSheet ssheet3 = wb2.createSheet("S3");
ssheet3.setRowGenerator((ssxSheet) -> {
Row row3_1 = ssxSheet.createRow(1);
Cell cell3_1_1 = row3_1.createCell(1);
cell3_1_1.setCellValue("value 3_1_1");
});
try (XSSFWorkbook xssfWb3 = DeferredSXSSFITestDataProvider.instance.inMemoryWriteOutAndReadBack(wb2)) {
assertEquals(3, xssfWb3.getNumberOfSheets());
// Verify sheet 1
XSSFSheet sheet1 = xssfWb3.getSheetAt(0);
assertEquals("S1", sheet1.getSheetName());
assertEquals(1, sheet1.getPhysicalNumberOfRows());
XSSFRow row1_1 = sheet1.getRow(1);
assertNotNull(row1_1);
XSSFCell cell1_1_1 = row1_1.getCell(1);
assertNotNull(cell1_1_1);
assertEquals("value 1_1_1", cell1_1_1.getStringCellValue());
// Verify sheet 2
XSSFSheet sheet2 = xssfWb3.getSheetAt(1);
assertEquals("S2", sheet2.getSheetName());
assertEquals(2, sheet2.getPhysicalNumberOfRows());
Row row2_1 = sheet2.getRow(1);
assertNotNull(row2_1);
Cell cell2_1_1 = row2_1.getCell(1);
assertNotNull(cell2_1_1);
assertEquals("value 2_1_1", cell2_1_1.getStringCellValue());
XSSFRow row2_2 = sheet2.getRow(2);
assertNotNull(row2_2);
XSSFCell cell2_2_1 = row2_2.getCell(1);
assertNotNull(cell2_2_1);
assertEquals("value 2_2_1", cell2_2_1.getStringCellValue());
// Verify sheet 3
XSSFSheet sheet3 = xssfWb3.getSheetAt(2);
assertEquals("S3", sheet3.getSheetName());
assertEquals(1, sheet3.getPhysicalNumberOfRows());
XSSFRow row3_1 = sheet3.getRow(1);
assertNotNull(row3_1);
XSSFCell cell3_1_1 = row3_1.getCell(1);
assertNotNull(cell3_1_1);
assertEquals("value 3_1_1", cell3_1_1.getStringCellValue());
}
}
}
}
}
@Test
void sheetdataWriter() throws IOException {
try (DeferredSXSSFWorkbook wb = new DeferredSXSSFWorkbook()) {