try to fix no-scratchpad build

remove or set-to-static scratchpad dependency from modules

git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1892795 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Andreas Beeker 2021-09-01 22:32:48 +00:00
parent 7adf9c26b7
commit 5c6493a5fc
20 changed files with 194 additions and 192 deletions

View File

@ -120,6 +120,8 @@ subprojects {
TEST9_SRC = 'src/test/java9' TEST9_SRC = 'src/test/java9'
TEST9_OUT = "${buildDir}/classes/java9/test/" TEST9_OUT = "${buildDir}/classes/java9/test/"
VERSIONS9 = 'META-INF/versions/9' VERSIONS9 = 'META-INF/versions/9'
NO_SCRATCHPAD = (findProperty("scratchpad.ignore") == "true")
} }
tasks.withType(JavaCompile) { tasks.withType(JavaCompile) {

View File

@ -63,7 +63,7 @@ def poijobs = [
], ],
[ name: 'POI-DSL-Gradle', trigger: triggerSundays, email: 'centic@apache.org', gradle: true [ name: 'POI-DSL-Gradle', trigger: triggerSundays, email: 'centic@apache.org', gradle: true
], ],
[ name: 'POI-DSL-no-scratchpad', trigger: triggerSundays, noScratchpad: true [ name: 'POI-DSL-no-scratchpad', trigger: triggerSundays, noScratchpad: true, gradle: true
], ],
[ name: 'POI-DSL-SonarQube', jdk: '1.11', trigger: 'H 7 * * *', maven: true, sonar: true, skipcigame: true, [ name: 'POI-DSL-SonarQube', jdk: '1.11', trigger: 'H 7 * * *', maven: true, sonar: true, skipcigame: true,
email: 'kiwiwings@apache.org', email: 'kiwiwings@apache.org',
@ -445,17 +445,9 @@ poijobs.each { poijob ->
gradle { gradle {
tasks('clean jenkins') tasks('clean jenkins')
useWrapper(true) useWrapper(true)
} if (poijob.noScratchpad) {
} else if (poijob.noScratchpad) { switches('-Pscratchpad.ignore=true')
ant { }
targets(['clean', 'compile'] + (poijob.properties ?: []))
prop('coverage.enabled', true)
antInstallation(antRT)
}
ant {
targets(['-Dscratchpad.ignore=true', 'jacocotask', 'test-all', 'testcoveragereport'] + (poijob.properties ?: []))
prop('coverage.enabled', true)
antInstallation(antRT)
} }
} else { } else {
ant { ant {

View File

@ -28,20 +28,29 @@ sourceSets {
dependencies { dependencies {
api project(':poi-ooxml') api project(':poi-ooxml')
api project(':poi-scratchpad') // api project(':poi-scratchpad')
implementation project(path: ':poi-ooxml', configuration: 'archives') implementation project(path: ':poi-ooxml', configuration: 'archives')
implementation project(path: ':poi-ooxml-full', configuration: 'archives') implementation project(path: ':poi-ooxml-full', configuration: 'archives')
implementation project(path: ':poi-scratchpad', configuration: 'archives')
if (NO_SCRATCHPAD) {
compileOnly project(path: ':poi-scratchpad', configuration: 'archives')
} else {
implementation project(path: ':poi-scratchpad', configuration: 'archives')
}
implementation "org.apache.logging.log4j:log4j-core:${log4jVersion}" implementation "org.apache.logging.log4j:log4j-core:${log4jVersion}"
testImplementation project(path: ':poi-ooxml', configuration: 'tests') testImplementation(project(path: ':poi-ooxml', configuration: 'tests')) {
if (NO_SCRATCHPAD) {
exclude group: 'org.apache.poi', module: 'poi-scratchpad'
}
}
testImplementation project(path: ':poi', configuration: 'tests') testImplementation project(path: ':poi', configuration: 'tests')
} }
final String MODULE_NAME = 'org.apache.poi.examples' final String MODULE_NAME = 'org.apache.poi.examples'
final Pattern MODULE_REGEX = ~'\\.jar$' final Pattern MODULE_REGEX = ~'\\.jar$'
final List MAIN_MODULE_PATH = sourceSets.main.runtimeClasspath.findAll{ it.path =~ MODULE_REGEX }.collect{ it.parent }.unique() final List MODULE_COMPILE_PATH = sourceSets.main.compileClasspath.findAll{ it.path =~ MODULE_REGEX }.collect{ it.parent }.unique()
task compileJava9(type: JavaCompile) { task compileJava9(type: JavaCompile) {
dependsOn 'compileJava', ':poi-ooxml:jar', ':poi-scratchpad:jar' dependsOn 'compileJava', ':poi-ooxml:jar', ':poi-scratchpad:jar'
@ -53,7 +62,7 @@ task compileJava9(type: JavaCompile) {
classpath = files() classpath = files()
options.compilerArgs = [ options.compilerArgs = [
'--patch-module', "${MODULE_NAME}=${sourceSets.main.output.classesDirs.asPath}", '--patch-module', "${MODULE_NAME}=${sourceSets.main.output.classesDirs.asPath}",
'--module-path', files(MAIN_MODULE_PATH).asPath '--module-path', files(MODULE_COMPILE_PATH).asPath
] ]
} }

View File

@ -18,7 +18,7 @@
module org.apache.poi.examples { module org.apache.poi.examples {
requires transitive org.apache.poi.ooxml; requires transitive org.apache.poi.ooxml;
requires transitive org.apache.poi.scratchpad; requires static org.apache.poi.scratchpad;
requires java.xml; requires java.xml;
exports org.apache.poi.examples.crypt; exports org.apache.poi.examples.crypt;

View File

@ -38,14 +38,13 @@ dependencies {
api 'org.apache.ant:ant:1.10.9' api 'org.apache.ant:ant:1.10.9'
api project(':poi-ooxml') api project(':poi-ooxml')
api project(':poi-scratchpad')
api project(path: ':poi-ooxml', configuration: 'archives') api project(path: ':poi-ooxml', configuration: 'archives')
api project(path: ':poi-scratchpad', configuration: 'archives')
testImplementation project(path: ':poi-ooxml-lite-agent', configuration: 'archives') testImplementation project(path: ':poi-ooxml-lite-agent', configuration: 'archives')
testImplementation project(path: ':poi', configuration: 'tests') testImplementation project(path: ':poi', configuration: 'tests')
testImplementation project(path: ':poi-ooxml', configuration: 'tests') testImplementation(project(path: ':poi-ooxml', configuration: 'tests')) {
testImplementation project(path: ':poi-scratchpad', configuration: 'tests') exclude group: 'org.apache.poi', module: 'poi-scratchpad'
}
testImplementation 'com.google.guava:guava:30.0-jre' testImplementation 'com.google.guava:guava:30.0-jre'
testImplementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.14.1' testImplementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.14.1'
} }

View File

@ -19,7 +19,6 @@ module org.apache.poi.excelant {
requires ant; requires ant;
requires org.apache.poi.ooxml; requires org.apache.poi.ooxml;
requires org.apache.poi.scratchpad;
exports org.apache.poi.ss.excelant; exports org.apache.poi.ss.excelant;
exports org.apache.poi.ss.excelant.util; exports org.apache.poi.ss.excelant.util;

View File

@ -19,7 +19,6 @@ module org.apache.poi.excelant {
requires ant; requires ant;
requires org.apache.poi.ooxml; requires org.apache.poi.ooxml;
requires org.apache.poi.scratchpad;
exports org.apache.poi.ss.excelant; exports org.apache.poi.ss.excelant;
exports org.apache.poi.ss.excelant.util; exports org.apache.poi.ss.excelant.util;

View File

@ -44,13 +44,34 @@ dependencies {
testImplementation 'org.apache.commons:commons-collections4:4.4' testImplementation 'org.apache.commons:commons-collections4:4.4'
testImplementation 'com.google.guava:guava:30.0-jre' testImplementation 'com.google.guava:guava:30.0-jre'
testImplementation project(':poi-ooxml') testImplementation(project(':poi-ooxml')) {
testImplementation project(':poi-scratchpad') if (NO_SCRATCHPAD) {
testImplementation project(':poi-examples') exclude group: 'org.apache.poi', module: 'poi-scratchpad'
testImplementation project(path:':poi-examples', configuration:'archives') }
}
if (NO_SCRATCHPAD) {
testCompileOnly project(':poi-scratchpad')
testCompileOnly project(path:':poi-scratchpad', configuration:'tests')
} else {
testImplementation project(':poi-scratchpad')
testImplementation project(path:':poi-scratchpad', configuration:'tests')
}
testImplementation(project(':poi-examples')) {
if (NO_SCRATCHPAD) {
exclude group: 'org.apache.poi', module: 'poi-scratchpad'
}
}
testImplementation(project(path:':poi-examples', configuration:'archives')) {
if (NO_SCRATCHPAD) {
exclude group: 'org.apache.poi', module: 'poi-scratchpad'
}
}
testImplementation project(path:':poi', configuration:'tests') testImplementation project(path:':poi', configuration:'tests')
testImplementation project(path:':poi-ooxml', configuration:'tests') testImplementation(project(path:':poi-ooxml', configuration:'tests')) {
testImplementation project(path:':poi-scratchpad', configuration:'tests') if (NO_SCRATCHPAD) {
exclude group: 'org.apache.poi', module: 'poi-scratchpad'
}
}
testImplementation project(path: ':poi-ooxml-lite-agent', configuration: 'archives') testImplementation project(path: ':poi-ooxml-lite-agent', configuration: 'archives')
testImplementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.14.1' testImplementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.14.1'
} }
@ -58,7 +79,8 @@ dependencies {
final String MODULE_NAME = 'org.apache.poi.stress' final String MODULE_NAME = 'org.apache.poi.stress'
final Pattern MODULE_NOT_REGEX = ~'((poi|poi-scratchpad|poi-ooxml)[/\\\\][^/\\\\]+$|batik-script)' final Pattern MODULE_NOT_REGEX = ~'((poi|poi-scratchpad|poi-ooxml)[/\\\\][^/\\\\]+$|batik-script)'
final Pattern MODULE_REGEX = ~'\\.jar$' final Pattern MODULE_REGEX = ~'\\.jar$'
final List TEST_MODULE_PATH = sourceSets.test.runtimeClasspath.findAll{ it.path =~ MODULE_REGEX && !(it.path =~ MODULE_NOT_REGEX) }.collect{ it.parent }.unique() final List MODULE_COMPILE_PATH = sourceSets.test.compileClasspath.findAll{ it.path =~ MODULE_REGEX && !(it.path =~ MODULE_NOT_REGEX) }.collect{ it.parent }.unique()
final List MODULE_RUNTIME_PATH = sourceSets.test.runtimeClasspath.findAll{ it.path =~ MODULE_REGEX && !(it.path =~ MODULE_NOT_REGEX) }.collect{ it.parent }.unique()
final String OOXML_LITE_AGENT = "../build/dist/maven/poi-ooxml-lite-agent/poi-ooxml-lite-agent-${project.version}.jar" final String OOXML_LITE_AGENT = "../build/dist/maven/poi-ooxml-lite-agent/poi-ooxml-lite-agent-${project.version}.jar"
final String OOXML_LITE_REPORT = '../build/ooxml-lite-report' final String OOXML_LITE_REPORT = '../build/ooxml-lite-report'
@ -79,7 +101,7 @@ task compileTest9(type: JavaCompile) {
source = file(TEST9_SRC) source = file(TEST9_SRC)
options.compilerArgs = [ options.compilerArgs = [
'--patch-module', "${MODULE_NAME}=${(sourceSets.main.output.classesDirs + sourceSets.test.output.classesDirs).asPath}", '--patch-module', "${MODULE_NAME}=${(sourceSets.main.output.classesDirs + sourceSets.test.output.classesDirs).asPath}",
'--module-path', files(TEST_MODULE_PATH).asPath '--module-path', files(MODULE_COMPILE_PATH).asPath
] ]
classpath = files() classpath = files()
} }
@ -141,9 +163,12 @@ test {
if (JavaVersion.current() != JavaVersion.VERSION_1_8) { if (JavaVersion.current() != JavaVersion.VERSION_1_8) {
jvmArgs += [ jvmArgs += [
'--add-modules', MODULE_NAME, '--add-modules', MODULE_NAME,
'--module-path', '../build/dist/maven/poi-integration-tests' + File.pathSeparator + files(TEST_MODULE_PATH).asPath, '--module-path', '../build/dist/maven/poi-integration-tests' + File.pathSeparator + files(MODULE_RUNTIME_PATH).asPath,
] ]
} }
if (NO_SCRATCHPAD) {
systemProperty 'scratchpad.ignore', 'true'
}
} }
} }

View File

@ -1,120 +0,0 @@
/* ====================================================================
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==================================================================== */
package org.apache.poi.stress;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;
public class FileHandlerFactory {
// map from patterns for mimetypes to the FileHandlers that should be able to
// work with that file
// use a Set<Pair> to have a defined order of applying the matches
private static final Map<Pattern, FileHandler> MIME_TYPES = new HashMap<>();
static {
////////////////// Word
MIME_TYPES.put(Pattern.compile("application/vnd.ms-word.document.macroenabled.12"), new XWPFFileHandler());
MIME_TYPES.put(Pattern.compile("application/vnd.ms-word.template.macroenabled.12"), new XWPFFileHandler());
// application/msword
MIME_TYPES.put(Pattern.compile(".*msword.*"), new HWPFFileHandler());
// application/vnd.ms-word
MIME_TYPES.put(Pattern.compile(".*ms-word.*"), new HWPFFileHandler());
// application/vnd.openxmlformats-officedocument.wordprocessingml.document
MIME_TYPES.put(Pattern.compile(".*wordprocessingml.*"), new XWPFFileHandler());
////////////////// Excel
MIME_TYPES.put(Pattern.compile("application/vnd.ms-excel.addin.macroEnabled.12"), new XSSFFileHandler());
MIME_TYPES.put(Pattern.compile("application/vnd.ms-excel.sheet.binary.macroEnabled.12"), new XSSFFileHandler());
// application/msexcel
MIME_TYPES.put(Pattern.compile(".*msexcel.*"), new HSSFFileHandler());
// application/vnd.ms-excel
MIME_TYPES.put(Pattern.compile(".*ms-excel.*"), new HSSFFileHandler());
// application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
MIME_TYPES.put(Pattern.compile(".*spreadsheetml.*"), new XSSFFileHandler());
////////////////// Powerpoint
// application/vnd.ms-powerpoint
MIME_TYPES.put(Pattern.compile("application/vnd.ms-powerpoint"), new HSLFFileHandler());
// application/vnd.ms-officetheme
MIME_TYPES.put(Pattern.compile("application/vnd.ms-officetheme"), new HSLFFileHandler());
// application/vnd.openxmlformats-officedocument.presentationml.presentation
MIME_TYPES.put(Pattern.compile(".*presentationml.*"), new XSLFFileHandler());
// application/vnd.ms-powerpoint.presentation.macroenabled.12
MIME_TYPES.put(Pattern.compile("application/vnd.ms-powerpoint.presentation.macroenabled.12"), new XSLFFileHandler());
// application/vnd.ms-powerpoint.slideshow.macroenabled.12
MIME_TYPES.put(Pattern.compile("application/vnd.ms-powerpoint.slideshow.macroenabled.12"), new XSLFFileHandler());
////////////////// Mail/TNEF
// application/vnd.ms-tnef
MIME_TYPES.put(Pattern.compile(".*ms-tnef.*"), new HMEFFileHandler());
// application/vnd.ms-outlook
MIME_TYPES.put(Pattern.compile("application/vnd.ms-outlook"), new HSMFFileHandler());
////////////////// Visio
// application/vnd.visio
MIME_TYPES.put(Pattern.compile("application/vnd.visio.*"), new HDGFFileHandler());
// application/vnd.ms-visio.drawing
MIME_TYPES.put(Pattern.compile(".*vnd.ms-visio\\."), new XDGFFileHandler());
//application/vnd.ms-visio.viewer
MIME_TYPES.put(Pattern.compile(".*visio.*"), new HDGFFileHandler());
////////////////// Publisher
// application/x-mspublisher
MIME_TYPES.put(Pattern.compile("application/x-mspublisher"), new HPBFFileHandler());
////////////////// Others
// special type used by Tika
MIME_TYPES.put(Pattern.compile("application/x-tika-ooxml.*"), new OPCFileHandler());
// special type used by Tika
MIME_TYPES.put(Pattern.compile("application/x-tika-msoffice.*"), new POIFSFileHandler());
// application/x-tika-old-excel
MIME_TYPES.put(Pattern.compile("application/x-tika-old-excel"), new POIFSFileHandler());
// application/vnd.openxmlformats-officedocument.drawingml.chart+xml
// ?!MIME_TYPES.put(Pattern.compile(".*drawingml.*"), ".dwg");
// application/vnd.openxmlformats-officedocument.vmlDrawing
// ?!MIME_TYPES.put(Pattern.compile(".*vmlDrawing.*"), ".dwg");
}
public static FileHandler getHandler(String mimeType) {
for(Map.Entry<Pattern,FileHandler> entry : MIME_TYPES.entrySet()) {
if(entry.getKey().matcher(mimeType).matches()) {
return entry.getValue();
}
}
return null;
}
}

View File

@ -22,29 +22,33 @@ import java.util.function.Supplier;
@SuppressWarnings("unused") @SuppressWarnings("unused")
public enum FileHandlerKnown { public enum FileHandlerKnown {
HDGF(HDGFFileHandler::new), HDGF,
HMEF(HMEFFileHandler::new), HMEF,
HPBF(HPBFFileHandler::new), HPBF,
HPSF(HPSFFileHandler::new), HPSF,
HSLF(HSLFFileHandler::new), HSLF,
HSMF(HSMFFileHandler::new), HSMF,
HSSF(HSSFFileHandler::new), HSSF,
HWPF(HWPFFileHandler::new), HWPF,
OPC(OPCFileHandler::new), OPC,
POIFS(POIFSFileHandler::new), POIFS,
XDGF(XDGFFileHandler::new), XDGF,
XSLF(XSLFFileHandler::new), XSLF,
XSSFB(XSSFBFileHandler::new), XSSFB,
XSSF(XSSFFileHandler::new), XSSF,
XWPF(XWPFFileHandler::new), XWPF,
OWPF(OWPFFileHandler::new), OWPF,
NULL(NullFileHandler::new) NULL
; ;
public final Supplier<FileHandler> fileHandler; public FileHandler getHandler() {
try {
FileHandlerKnown(Supplier<FileHandler> fileHandler) { // Because of no-scratchpad handling, we need to resort to reflection here
this.fileHandler = fileHandler; String n = name().replace("NULL", "Null");
return (FileHandler)Class.forName("org.apache.poi.stress." + n + "FileHandler").newInstance();
} catch (Exception e) {
return new NullFileHandler();
}
} }
private static class NullFileHandler implements FileHandler { private static class NullFileHandler implements FileHandler {

View File

@ -18,6 +18,8 @@ package org.apache.poi.stress;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.jupiter.api.Assumptions.assumeFalse; import static org.junit.jupiter.api.Assumptions.assumeFalse;
@ -31,10 +33,12 @@ import java.util.Set;
import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
import org.apache.poi.examples.hpsf.CopyCompare; import org.apache.poi.examples.hpsf.CopyCompare;
import org.apache.poi.extractor.POITextExtractor;
import org.apache.poi.hpsf.DocumentSummaryInformation; import org.apache.poi.hpsf.DocumentSummaryInformation;
import org.apache.poi.hpsf.HPSFPropertiesOnlyDocument; import org.apache.poi.hpsf.HPSFPropertiesOnlyDocument;
import org.apache.poi.hpsf.PropertySet; import org.apache.poi.hpsf.PropertySet;
import org.apache.poi.hpsf.SummaryInformation; import org.apache.poi.hpsf.SummaryInformation;
import org.apache.poi.hpsf.extractor.HPSFPropertiesExtractor;
import org.apache.poi.poifs.filesystem.DirectoryNode; import org.apache.poi.poifs.filesystem.DirectoryNode;
import org.apache.poi.poifs.filesystem.DocumentInputStream; import org.apache.poi.poifs.filesystem.DocumentInputStream;
import org.apache.poi.poifs.filesystem.POIFSFileSystem; import org.apache.poi.poifs.filesystem.POIFSFileSystem;
@ -56,6 +60,35 @@ class HPSFFileHandler extends POIFSFileHandler {
"document/word2.doc" "document/word2.doc"
); );
@Override
public void handleExtracting(File file) throws Exception {
if (!Boolean.getBoolean("scratchpad.ignore")) {
super.handleExtracting(file);
return;
}
long length = file.length();
long modified = file.lastModified();
try (POIFSFileSystem poifs = new POIFSFileSystem(file);
HPSFPropertiesExtractor extractor = new HPSFPropertiesExtractor(poifs)) {
String fileAndParentName = file.getParentFile().getName() + "/" + file.getName();
String relPath = file.getPath().replaceAll(".*test-data", "test-data").replace('\\', '/');
assertFalse(EXPECTED_EXTRACTOR_FAILURES.contains(fileAndParentName),
"Expected Extraction to fail for file " + relPath + " and handler " + this + ", but did not fail!");
assertNotNull(extractor.getDocumentSummaryInformationText());
assertNotNull(extractor.getSummaryInformationText());
String text = extractor.getText();
//System.out.println(text);
assertNotNull(text);
}
assertEquals(length, file.length(), "File should not be modified by extractor");
assertEquals(modified, file.lastModified(), "File should not be modified by extractor");
}
@Override @Override
public void handleFile(InputStream stream, String path) throws Exception { public void handleFile(InputStream stream, String path) throws Exception {
POIFSFileSystem poifs = new POIFSFileSystem(stream); POIFSFileSystem poifs = new POIFSFileSystem(stream);

View File

@ -18,12 +18,14 @@ package org.apache.poi.stress;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.AbstractMap.SimpleEntry;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.function.BiConsumer; import java.util.function.BiConsumer;
import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.stream.StreamSupport; import java.util.stream.StreamSupport;
@ -40,7 +42,8 @@ import org.apache.poi.ss.usermodel.WorkbookFactory;
public class StressMap { public class StressMap {
private final MultiValuedMap<String, ExcInfo> exMap = new ArrayListValuedHashMap<>(); private final MultiValuedMap<String, ExcInfo> exMap = new ArrayListValuedHashMap<>();
private final Map<String,String> handlerMap = new LinkedHashMap<>(); private final Map<String,String> handlerMap = new LinkedHashMap<>();
private final boolean SCRATCH_IGNORE = Boolean.getBoolean("scratchpad.ignore");
private final Pattern SCRATCH_HANDLER = Pattern.compile("(HSLF|HWPF|HSMF|HMEF)");
public void load(File mapFile) throws IOException { public void load(File mapFile) throws IOException {
try (Workbook wb = WorkbookFactory.create(mapFile)) { try (Workbook wb = WorkbookFactory.create(mapFile)) {
@ -79,7 +82,6 @@ public class StressMap {
handlerMap.clear(); handlerMap.clear();
boolean IGNORE_SCRATCHPAD = Boolean.getBoolean("scratchpad.ignore");
boolean isFirst = true; boolean isFirst = true;
for (Row row : sh) { for (Row row : sh) {
if (isFirst) { if (isFirst) {
@ -87,7 +89,7 @@ public class StressMap {
continue; continue;
} }
Cell cell = row.getCell(2); Cell cell = row.getCell(2);
if (IGNORE_SCRATCHPAD || cell == null || cell.getCellType() != CellType.STRING) { if (SCRATCH_IGNORE || cell == null || cell.getCellType() != CellType.STRING) {
cell = row.getCell(1); cell = row.getCell(1);
} }
handlerMap.put(row.getCell(0).getStringCellValue(), cell.getStringCellValue()); handlerMap.put(row.getCell(0).getStringCellValue(), cell.getStringCellValue());
@ -103,20 +105,38 @@ public class StressMap {
exMap.clear(); exMap.clear();
Iterator<Row> iter = sh.iterator(); Iterator<Row> iter = sh.iterator();
List<BiConsumer<ExcInfo,String>> cols = initCols(iter.next()); List<Map.Entry<String, BiConsumer<ExcInfo,String>>> cols = initCols(iter.next());
int idx = 0, handlerIdx = -1;
for (Map.Entry<String, BiConsumer<ExcInfo, String>> e : cols) {
if ("Handler".equals(e.getKey())) {
handlerIdx = idx;
}
idx++;
}
while (iter.hasNext()) { while (iter.hasNext()) {
Row row = iter.next();
if (SCRATCH_IGNORE && handlerIdx > -1) {
String handler = row.getCell(handlerIdx).getStringCellValue();
if (SCRATCH_HANDLER.matcher(handler).find()) {
// ignore exception of ignored files
continue;
}
}
ExcInfo info = new ExcInfo(); ExcInfo info = new ExcInfo();
for (Cell cell : iter.next()) { for (Cell cell : row) {
if (cell.getCellType() == CellType.STRING) { if (cell.getCellType() == CellType.STRING) {
cols.get(cell.getColumnIndex()).accept(info, cell.getStringCellValue()); cols.get(cell.getColumnIndex()).getValue().accept(info, cell.getStringCellValue());
} }
} }
exMap.put(info.getFile(), info); exMap.put(info.getFile(), info);
} }
} }
private static List<BiConsumer<ExcInfo,String>> initCols(Row row) { private static List<Map.Entry<String, BiConsumer<ExcInfo,String>>> initCols(Row row) {
Map<String,BiConsumer<ExcInfo,String>> m = new HashMap<>(); Map<String,BiConsumer<ExcInfo,String>> m = new HashMap<>();
m.put("File", ExcInfo::setFile); m.put("File", ExcInfo::setFile);
m.put("Tests", ExcInfo::setTests); m.put("Tests", ExcInfo::setTests);
@ -128,7 +148,7 @@ public class StressMap {
return StreamSupport return StreamSupport
.stream(row.spliterator(), false) .stream(row.spliterator(), false)
.map(Cell::getStringCellValue) .map(Cell::getStringCellValue)
.map(v -> m.getOrDefault(v, (e,s) -> {})) .map(v -> new SimpleEntry<>(v, m.getOrDefault(v, (e,s) -> {})))
.collect(Collectors.toList()); .collect(Collectors.toList());
} }

View File

@ -66,7 +66,7 @@ import org.opentest4j.AssertionFailedError;
* that we do not remove expected sanity checks. * that we do not remove expected sanity checks.
*/ */
// also need to set JVM parameter: -Djunit.jupiter.execution.parallel.enabled=true // also need to set JVM parameter: -Djunit.jupiter.execution.parallel.enabled=true
@Execution(ExecutionMode.CONCURRENT) //@Execution(ExecutionMode.CONCURRENT)
public class TestAllFiles { public class TestAllFiles {
private static final String DEFAULT_TEST_DATA_PATH = "test-data"; private static final String DEFAULT_TEST_DATA_PATH = "test-data";
public static final File ROOT_DIR = new File(System.getProperty("POI.testdata.path", DEFAULT_TEST_DATA_PATH)); public static final File ROOT_DIR = new File(System.getProperty("POI.testdata.path", DEFAULT_TEST_DATA_PATH));
@ -78,6 +78,27 @@ public class TestAllFiles {
"**/right-to-left.xlsx" //the threaded comments in this file cause XSSF clone to fail "**/right-to-left.xlsx" //the threaded comments in this file cause XSSF clone to fail
}; };
// cheap workaround of skipping the few problematic files
public static final String[] SCAN_EXCLUDES_NOSCRATCHPAD = {
"**/.svn/**",
"lost+found",
"**/.git/**",
"**/right-to-left.xlsx",
"document/word2.doc",
"document/cpansearch.perl.org_src_tobyink_acme-rundoc-0.001_word-lib_hello_world.docm",
"hpsf/Test0313rur.adm",
"spreadsheet/43493.xls",
"spreadsheet/44958.xls",
"spreadsheet/44958_1.xls",
"spreadsheet/46904.xls",
"spreadsheet/51832.xls",
"spreadsheet/60284.xls",
"spreadsheet/testArraysAndTables.xls",
"spreadsheet/testEXCEL_3.xls",
"spreadsheet/testEXCEL_4.xls",
"poifs/unknown_properties.msg"
};
private static final Set<String> EXPECTED_FAILURES = StressTestUtils.unmodifiableHashSet( private static final Set<String> EXPECTED_FAILURES = StressTestUtils.unmodifiableHashSet(
"document/truncated62886.docx" "document/truncated62886.docx"
); );
@ -86,10 +107,11 @@ public class TestAllFiles {
StressMap sm = new StressMap(); StressMap sm = new StressMap();
sm.load(new File(ROOT_DIR, "spreadsheet/stress.xls")); sm.load(new File(ROOT_DIR, "spreadsheet/stress.xls"));
boolean noScratch = Boolean.getBoolean("scratchpad.ignore");
DirectoryScanner scanner = new DirectoryScanner(); DirectoryScanner scanner = new DirectoryScanner();
scanner.setBasedir(ROOT_DIR); scanner.setBasedir(ROOT_DIR);
scanner.setExcludes(SCAN_EXCLUDES); scanner.setExcludes(noScratch ? SCAN_EXCLUDES_NOSCRATCHPAD : SCAN_EXCLUDES);
scanner.scan(); scanner.scan();
final List<Arguments> result = new ArrayList<>(100); final List<Arguments> result = new ArrayList<>(100);
@ -121,7 +143,7 @@ public class TestAllFiles {
if (StressTestUtils.excludeFile(file, EXPECTED_FAILURES)) return; if (StressTestUtils.excludeFile(file, EXPECTED_FAILURES)) return;
System.out.println("Running extractFiles on "+file); System.out.println("Running extractFiles on "+file);
FileHandler fileHandler = handler.fileHandler.get(); FileHandler fileHandler = handler.getHandler();
assertNotNull(fileHandler, "Did not find a handler for file " + file); assertNotNull(fileHandler, "Did not find a handler for file " + file);
Executable exec = () -> fileHandler.handleExtracting(new File(ROOT_DIR, file)); Executable exec = () -> fileHandler.handleExtracting(new File(ROOT_DIR, file));
verify(file, exec, exClass, exMessage, password); verify(file, exec, exClass, exMessage, password);
@ -135,7 +157,7 @@ public class TestAllFiles {
@MethodSource("handleFiles") @MethodSource("handleFiles")
void handleFile(String file, FileHandlerKnown handler, String password, Class<? extends Throwable> exClass, String exMessage) throws IOException { void handleFile(String file, FileHandlerKnown handler, String password, Class<? extends Throwable> exClass, String exMessage) throws IOException {
System.out.println("Running handleFiles on "+file); System.out.println("Running handleFiles on "+file);
FileHandler fileHandler = handler.fileHandler.get(); FileHandler fileHandler = handler.getHandler();
assertNotNull(fileHandler, "Did not find a handler for file " + file); assertNotNull(fileHandler, "Did not find a handler for file " + file);
try (InputStream stream = new BufferedInputStream(new FileInputStream(new File(ROOT_DIR, file)), 64 * 1024)) { try (InputStream stream = new BufferedInputStream(new FileInputStream(new File(ROOT_DIR, file)), 64 * 1024)) {
Executable exec = () -> fileHandler.handleFile(stream, file); Executable exec = () -> fileHandler.handleFile(stream, file);
@ -151,7 +173,7 @@ public class TestAllFiles {
@MethodSource("handleAdditionals") @MethodSource("handleAdditionals")
void handleAdditional(String file, FileHandlerKnown handler, String password, Class<? extends Throwable> exClass, String exMessage) { void handleAdditional(String file, FileHandlerKnown handler, String password, Class<? extends Throwable> exClass, String exMessage) {
System.out.println("Running additionals on "+file); System.out.println("Running additionals on "+file);
FileHandler fileHandler = handler.fileHandler.get(); FileHandler fileHandler = handler.getHandler();
assertNotNull(fileHandler, "Did not find a handler for file " + file); assertNotNull(fileHandler, "Did not find a handler for file " + file);
Executable exec = () -> fileHandler.handleAdditional(new File(ROOT_DIR, file)); Executable exec = () -> fileHandler.handleAdditional(new File(ROOT_DIR, file));
verify(file, exec, exClass, exMessage, password); verify(file, exec, exClass, exMessage, password);

View File

@ -24,6 +24,7 @@ module org.apache.poi.stress {
requires org.apache.commons.collections4; requires org.apache.commons.collections4;
requires org.apache.poi.examples; requires org.apache.poi.examples;
requires static org.apache.poi.scratchpad;
exports org.apache.poi.stress; exports org.apache.poi.stress;

View File

@ -64,11 +64,12 @@ dependencies {
api 'de.rototor.pdfbox:graphics2d:0.32' api 'de.rototor.pdfbox:graphics2d:0.32'
if (!NO_SCRATCHPAD) {
testImplementation project(':poi-scratchpad') testImplementation project(':poi-scratchpad')
testImplementation project(path:':poi-scratchpad', configuration:'tests')
}
testImplementation project(path:':poi', configuration:'tests') testImplementation project(path:':poi', configuration:'tests')
testImplementation project(path:':poi-ooxml-lite-agent', configuration: 'archives') testImplementation project(path:':poi-ooxml-lite-agent', configuration: 'archives')
testImplementation project(path:':poi-scratchpad', configuration:'tests')
testImplementation 'org.xmlunit:xmlunit-core:2.8.0' testImplementation 'org.xmlunit:xmlunit-core:2.8.0'
testImplementation 'org.reflections:reflections:0.9.12' testImplementation 'org.reflections:reflections:0.9.12'
testImplementation 'org.openjdk.jmh:jmh-core:1.26' testImplementation 'org.openjdk.jmh:jmh-core:1.26'
@ -219,6 +220,12 @@ test {
systemProperties['junit.jupiter.execution.parallel.enabled'] = 'true' systemProperties['junit.jupiter.execution.parallel.enabled'] = 'true'
if (NO_SCRATCHPAD) {
useJUnitPlatform {
excludeTags 'scratchpad.ignore'
}
}
doFirst { doFirst {
jvmArgs += [ jvmArgs += [
"-javaagent:${OOXML_LITE_AGENT}=${OOXML_LITE_REPORT}|${OOXML_LITE_INCLUDES}", "-javaagent:${OOXML_LITE_AGENT}=${OOXML_LITE_REPORT}|${OOXML_LITE_INCLUDES}",
@ -226,7 +233,7 @@ test {
if (JavaVersion.current() != JavaVersion.VERSION_1_8) { if (JavaVersion.current() != JavaVersion.VERSION_1_8) {
jvmArgs += [ jvmArgs += [
'--add-modules', MODULE_NAME, '--add-modules', MODULE_NAME,
'--module-path', '../build/dist/maven/poi-ooxml-tests' + java.io.File.pathSeparator + files(TEST_MODULE_PATH).asPath, '--module-path', '../build/dist/maven/poi-ooxml-tests' + File.pathSeparator + files(TEST_MODULE_PATH).asPath,
] ]
} }
} }

View File

@ -45,6 +45,7 @@ import org.apache.poi.poifs.filesystem.FileMagic;
import org.apache.poi.poifs.filesystem.POIFSFileSystem; import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import org.apache.poi.xssf.extractor.XSSFEventBasedExcelExtractor; import org.apache.poi.xssf.extractor.XSSFEventBasedExcelExtractor;
import org.apache.poi.xssf.extractor.XSSFExcelExtractor; import org.apache.poi.xssf.extractor.XSSFExcelExtractor;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.Arguments;
@ -54,6 +55,7 @@ import org.junit.jupiter.params.provider.ValueSource;
/** /**
* Test that the extractor factory plays nicely * Test that the extractor factory plays nicely
*/ */
@Tag("scratchpad.ignore")
class TestExtractorFactory { class TestExtractorFactory {
private static final POIDataSamples ssTests = POIDataSamples.getSpreadSheetInstance(); private static final POIDataSamples ssTests = POIDataSamples.getSpreadSheetInstance();

View File

@ -160,6 +160,7 @@ import org.etsi.uri.x01903.v13.QualifyingPropertiesType;
import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource; import org.junit.jupiter.params.provider.ValueSource;
@ -924,6 +925,7 @@ class TestSignatureInfo {
@Test @Test
@DisabledOnJreEx("1.8.0_292") @DisabledOnJreEx("1.8.0_292")
@Tag("scratchpad.ignore")
void testSignatureImage() throws Exception { void testSignatureImage() throws Exception {
initKeyPair(); initKeyPair();

View File

@ -42,11 +42,13 @@ import org.apache.poi.poifs.crypt.EncryptionInfo;
import org.apache.poi.poifs.crypt.EncryptionMode; import org.apache.poi.poifs.crypt.EncryptionMode;
import org.apache.poi.poifs.crypt.cryptoapi.CryptoAPIEncryptionHeader; import org.apache.poi.poifs.crypt.cryptoapi.CryptoAPIEncryptionHeader;
import org.apache.poi.poifs.storage.RawDataUtil; import org.apache.poi.poifs.storage.RawDataUtil;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.MethodSource;
@Tag("scratchpad.ignore")
class TestHxxFEncryption { class TestHxxFEncryption {
public static Stream<Arguments> data() throws IOException { public static Stream<Arguments> data() throws IOException {
final String base64 = final String base64 =

View File

@ -63,8 +63,10 @@ import org.apache.poi.xssf.XSSFTestDataSamples;
import org.apache.poi.xssf.usermodel.XSSFObjectData; import org.apache.poi.xssf.usermodel.XSSFObjectData;
import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@Tag("scratchpad.ignore")
class TestEmbedOLEPackage { class TestEmbedOLEPackage {
private static byte[] samplePPT, samplePPTX, samplePNG; private static byte[] samplePPT, samplePPTX, samplePNG;

View File

@ -111,6 +111,7 @@ import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.extensions.XSSFCellFill; import org.apache.poi.xssf.usermodel.extensions.XSSFCellFill;
import org.apache.xmlbeans.XmlException; import org.apache.xmlbeans.XmlException;
import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.CsvSource;
@ -3439,6 +3440,7 @@ public final class TestXSSFBugs extends BaseTestBugzillaIssues {
@Test @Test
@Tag("scratchpad.ignore")
void testXLSXinPPT() throws Exception { void testXLSXinPPT() throws Exception {
assumeFalse(Boolean.getBoolean("scratchpad.ignore")); assumeFalse(Boolean.getBoolean("scratchpad.ignore"));