Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-929: contrib/benchmark build doesn't handle checking if content is properly extracted (trunk) 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1063647 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Shai Erera 2011-01-26 09:10:06 +00:00
parent c403d3e756
commit e76ad0990d
3 changed files with 110 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
modules/benchmark/CHANGES.txt
View File

@ -2,6 +2,11 @@ Lucene Benchmark Contrib Change Log
The Benchmark contrib package contains code for benchmarking Lucene in a variety of ways.
01/26/2011
LUCENE-929: ExtractReuters first extracts to a tmp dir and then renames. That
way, if a previous extract attempt failed, "ant extract-reuters" will still
extract the files. (Shai Erera, Doron Cohen, Grant Ingersoll)
01/24/2011
LUCENE-2885: Add WaitForMerges task (calls IndexWriter.waitForMerges()).
(Mike McCandless)

1
modules/benchmark/build.xml
View File

@ -87,7 +87,6 @@
</target>
<target name="extract-reuters" depends="check-files" unless="reuters.extracted">
<mkdir dir="${working.dir}/reuters-out"/>
<java classname="org.apache.lucene.benchmark.utils.ExtractReuters" maxmemory="1024M" fork="true">
<classpath refid="run.classpath"/>
<arg file="${working.dir}/reuters"/>

237
modules/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java
View File

@ -29,146 +29,119 @@ import java.util.regex.Pattern;
/**
* Split the Reuters SGML documents into Simple Text files containing: Title, Date, Dateline, Body
*/
public class ExtractReuters
{
private File reutersDir;
private File outputDir;
private static final String LINE_SEPARATOR = System.getProperty("line.separator");
public ExtractReuters(File reutersDir, File outputDir)
{
this.reutersDir = reutersDir;
this.outputDir = outputDir;
System.out.println("Deleting all files in " + outputDir);
File [] files = outputDir.listFiles();
for (int i = 0; i < files.length; i++)
{
files[i].delete();
}
public class ExtractReuters {
private File reutersDir;
private File outputDir;
private static final String LINE_SEPARATOR = System.getProperty("line.separator");
public ExtractReuters(File reutersDir, File outputDir) {
this.reutersDir = reutersDir;
this.outputDir = outputDir;
System.out.println("Deleting all files in " + outputDir);
for (File f : outputDir.listFiles()) {
f.delete();
}
}
public void extract()
{
File [] sgmFiles = reutersDir.listFiles(new FileFilter()
{
public boolean accept(File file)
{
return file.getName().endsWith(".sgm");
}
});
if (sgmFiles != null && sgmFiles.length > 0)
{
for (int i = 0; i < sgmFiles.length; i++)
{
File sgmFile = sgmFiles[i];
extractFile(sgmFile);
public void extract() {
File[] sgmFiles = reutersDir.listFiles(new FileFilter() {
public boolean accept(File file) {
return file.getName().endsWith(".sgm");
}
});
if (sgmFiles != null && sgmFiles.length > 0) {
for (File sgmFile : sgmFiles) {
extractFile(sgmFile);
}
} else {
System.err.println("No .sgm files in " + reutersDir);
}
}
Pattern EXTRACTION_PATTERN = Pattern
.compile("<TITLE>(.*?)</TITLE>|<DATE>(.*?)</DATE>|<BODY>(.*?)</BODY>");
private static String[] META_CHARS = { "&", "<", ">", "\"", "'" };
private static String[] META_CHARS_SERIALIZATIONS = { "&amp;", "&lt;",
"&gt;", "&quot;", "&apos;" };
/**
* Override if you wish to change what is extracted
*
* @param sgmFile
*/
protected void extractFile(File sgmFile) {
try {
BufferedReader reader = new BufferedReader(new FileReader(sgmFile));
StringBuilder buffer = new StringBuilder(1024);
StringBuilder outBuffer = new StringBuilder(1024);
String line = null;
int docNumber = 0;
while ((line = reader.readLine()) != null) {
// when we see a closing reuters tag, flush the file
if (line.indexOf("</REUTERS") == -1) {
// Replace the SGM escape sequences
buffer.append(line).append(' ');// accumulate the strings for now,
// then apply regular expression to
// get the pieces,
} else {
// Extract the relevant pieces and write to a file in the output dir
Matcher matcher = EXTRACTION_PATTERN.matcher(buffer);
while (matcher.find()) {
for (int i = 1; i <= matcher.groupCount(); i++) {
if (matcher.group(i) != null) {
outBuffer.append(matcher.group(i));
}
}
outBuffer.append(LINE_SEPARATOR).append(LINE_SEPARATOR);
}
String out = outBuffer.toString();
for (int i = 0; i < META_CHARS_SERIALIZATIONS.length; i++) {
out = out.replaceAll(META_CHARS_SERIALIZATIONS[i], META_CHARS[i]);
}
File outFile = new File(outputDir, sgmFile.getName() + "-"
+ (docNumber++) + ".txt");
// System.out.println("Writing " + outFile);
FileWriter writer = new FileWriter(outFile);
writer.write(out);
writer.close();
outBuffer.setLength(0);
buffer.setLength(0);
}
else
{
System.err.println("No .sgm files in " + reutersDir);
}
}
reader.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
Pattern EXTRACTION_PATTERN = Pattern.compile("<TITLE>(.*?)</TITLE>|<DATE>(.*?)</DATE>|<BODY>(.*?)</BODY>");
private static String[] META_CHARS
= {"&", "<", ">", "\"", "'"};
private static String[] META_CHARS_SERIALIZATIONS
= {"&amp;", "&lt;", "&gt;", "&quot;", "&apos;"};
/**
* Override if you wish to change what is extracted
*
* @param sgmFile
*/
protected void extractFile(File sgmFile)
{
try
{
BufferedReader reader = new BufferedReader(new FileReader(sgmFile));
StringBuilder buffer = new StringBuilder(1024);
StringBuilder outBuffer = new StringBuilder(1024);
String line = null;
int docNumber = 0;
while ((line = reader.readLine()) != null)
{
//when we see a closing reuters tag, flush the file
if (line.indexOf("</REUTERS") == -1) {
//Replace the SGM escape sequences
buffer.append(line).append(' ');//accumulate the strings for now, then apply regular expression to get the pieces,
}
else
{
//Extract the relevant pieces and write to a file in the output dir
Matcher matcher = EXTRACTION_PATTERN.matcher(buffer);
while (matcher.find())
{
for (int i = 1; i <= matcher.groupCount(); i++)
{
if (matcher.group(i) != null)
{
outBuffer.append(matcher.group(i));
}
}
outBuffer.append(LINE_SEPARATOR).append(LINE_SEPARATOR);
}
String out = outBuffer.toString();
for (int i = 0; i < META_CHARS_SERIALIZATIONS.length; i++)
{
out = out.replaceAll(META_CHARS_SERIALIZATIONS[i], META_CHARS[i]);
}
File outFile = new File(outputDir, sgmFile.getName() + "-" + (docNumber++) + ".txt");
//System.out.println("Writing " + outFile);
FileWriter writer = new FileWriter(outFile);
writer.write(out);
writer.close();
outBuffer.setLength(0);
buffer.setLength(0);
}
}
reader.close();
}
catch (
IOException e
)
{
throw new RuntimeException(e);
}
public static void main(String[] args) {
if (args.length != 2) {
printUsage();
}
public static void main(String[] args)
{
if (args.length != 2)
{
printUsage();
}
File reutersDir = new File(args[0]);
if (reutersDir.exists())
{
File outputDir = new File(args[1]);
outputDir.mkdirs();
ExtractReuters extractor = new ExtractReuters(reutersDir, outputDir);
extractor.extract();
}
else
{
printUsage();
}
File reutersDir = new File(args[0]);
if (!reutersDir.exists()) {
printUsage();
return;
}
// First, extract to a tmp directory and only if everything succeeds, rename
// to output directory.
File outputDir = new File(args[1] + "-tmp");
outputDir.mkdirs();
ExtractReuters extractor = new ExtractReuters(reutersDir, outputDir);
extractor.extract();
// Now rename to requested output dir
outputDir.renameTo(new File(args[1]));
}
private static void printUsage()
{
System.err.println("Usage: java -cp <...> org.apache.lucene.benchmark.utils.ExtractReuters <Path to Reuters SGM files> <Output Path>");
}
private static void printUsage() {
System.err.println("Usage: java -cp <...> org.apache.lucene.benchmark.utils.ExtractReuters <Path to Reuters SGM files> <Output Path>");
}
}