Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-9768: Add source sets for src/tools, clean up forbidden API and formatting errors (#2361)

This commit is contained in:
Dawid Weiss 2021-02-12 17:03:54 +01:00 committed by GitHub
parent d89cb72fa7
commit f7e42bdb35
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 119 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
gradle/ant-compat/folder-layout.gradle
View File

@ -41,3 +41,23 @@ configure(project(":solr:webapp")) {
webAppDirName = "web"
}
}
allprojects {
plugins.withType(JavaPlugin) {
// if 'src/tools' exists, add it as a separate sourceSet.
if (file('src/tools/java').exists()) {
sourceSets {
tools {
java {
srcDirs = ['src/tools/java']
}
}
}
configurations {
// Inherit any dependencies from the main source set.
toolsImplementation.extendsFrom implementation
}
}
}
}

17
gradle/validation/forbidden-apis.gradle
View File

@ -89,6 +89,23 @@ allprojects { prj ->
]
}
// Configure defaults for sourceSets.tools (if present).
tasks.matching { it.name == "forbiddenApisTools" }.all {
bundledSignatures += [
'jdk-unsafe',
'jdk-deprecated',
'jdk-non-portable',
'jdk-reflection',
]
suppressAnnotations += [
"**.SuppressForbidden"
]
doFirst dynamicSignatures.curry(configurations.toolsCompileClasspath, "lucene")
inputs.dir(file(resources))
}
// Disable sysout signatures for these projects.
if (prj.path in [
":lucene:demo",

10
gradle/validation/rat-sources.gradle
View File

@ -154,10 +154,16 @@ class RatTask extends DefaultTask {
}
if (project.plugins.findPlugin(JavaPlugin)) {
[
def checkSets = [
project.sourceSets.main.java.srcDirs,
project.sourceSets.test.java.srcDirs,
].flatten().each { srcLocation ->
]
project.sourceSets.matching { it.name == 'tools' }.all {
checkSets += project.sourceSets.tools.java.srcDirs
}
checkSets.flatten().each { srcLocation ->
ant.fileset(dir: srcLocation, erroronmissingdir: false) {
srcExcludes.each { pattern -> ant.exclude(name: pattern) }
}

16
lucene/analysis/common/src/tools/java/org/apache/lucene/analysis/standard/GenerateJflexTLDMacros.java
View File

@ -17,8 +17,6 @@
package org.apache.lucene.analysis.standard;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
@ -26,6 +24,9 @@ import java.io.Writer;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.DateFormat;
import java.util.ArrayList;
import java.util.Comparator;
@ -102,7 +103,7 @@ public class GenerateJflexTLDMacros {
Pattern.compile("([-A-Za-z0-9]+)\\.\\s+\\d+\\s+IN\\s+NS\\s+.*");
private final URL tldFileURL;
private long tldFileLastModified = -1L;
private final File outputFile;
private final Path outputFile;
private final SortedMap<String, Boolean> processedTLDsLongestFirst =
new TreeMap<>(
Comparator.comparing(String::length).reversed().thenComparing(String::compareTo));
@ -111,7 +112,7 @@ public class GenerateJflexTLDMacros {
public GenerateJflexTLDMacros(String tldFileURL, String outputFile) throws Exception {
this.tldFileURL = new URL(tldFileURL);
this.outputFile = new File(outputFile);
this.outputFile = Paths.get(outputFile);
}
/**
@ -130,9 +131,10 @@ public class GenerateJflexTLDMacros {
for (int suffixLength = 0; suffixLength < TLDsBySuffixLength.size(); ++suffixLength) {
int domainsAtThisSuffixLength = TLDsBySuffixLength.get(suffixLength).size();
totalDomains += domainsAtThisSuffixLength;
System.out.printf("%30s: %4d TLDs%n", getMacroName(suffixLength), domainsAtThisSuffixLength);
System.out.printf(
Locale.ROOT, "%30s: %4d TLDs%n", getMacroName(suffixLength), domainsAtThisSuffixLength);
}
System.out.printf("%30s: %4d TLDs%n", "Total", totalDomains);
System.out.printf(Locale.ROOT, "%30s: %4d TLDs%n", "Total", totalDomains);
}
/**
@ -216,7 +218,7 @@ public class GenerateJflexTLDMacros {
DateFormat.getDateTimeInstance(DateFormat.FULL, DateFormat.FULL, Locale.ROOT);
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
try (Writer writer =
new OutputStreamWriter(new FileOutputStream(outputFile), StandardCharsets.UTF_8)) {
new OutputStreamWriter(Files.newOutputStream(outputFile), StandardCharsets.UTF_8)) {
writer.write(APACHE_LICENSE);
writer.write("// Generated from IANA Root Zone Database <");
writer.write(tldFileURL.toString());

84
lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java
View File

@ -21,10 +21,6 @@ import com.ibm.icu.lang.UProperty;
import com.ibm.icu.text.UnicodeSet;
import com.ibm.icu.text.UnicodeSetIterator;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@ -34,11 +30,16 @@ import java.io.Writer;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* Downloads/generates lucene/analysis/icu/src/data/utr30/*.txt
@ -82,33 +83,33 @@ public class GenerateUTR30DataFiles {
}
private static void expandRulesInUTR30DataFiles() throws IOException {
FileFilter filter =
new FileFilter() {
@Override
public boolean accept(File pathname) {
String name = pathname.getName();
return pathname.isFile()
&& name.matches(".*\\.(?s:txt)")
&& !name.equals(NFC_TXT)
&& !name.equals(NFKC_TXT)
&& !name.equals(NFKC_CF_TXT);
}
Predicate<Path> predicate =
(path) -> {
String name = path.getFileName().toString();
return Files.isRegularFile(path)
&& name.matches(".*\\.(?s:txt)")
&& !name.equals(NFC_TXT)
&& !name.equals(NFKC_TXT)
&& !name.equals(NFKC_CF_TXT);
};
for (File file : new File(".").listFiles(filter)) {
expandDataFileRules(file);
try (var stream = Files.list(Paths.get(".")).filter(predicate)) {
for (Path file : stream.collect(Collectors.toList())) {
expandDataFileRules(file);
}
}
}
private static void expandDataFileRules(File file) throws IOException {
final FileInputStream stream = new FileInputStream(file);
final InputStreamReader reader = new InputStreamReader(stream, StandardCharsets.UTF_8);
final BufferedReader bufferedReader = new BufferedReader(reader);
StringBuilder builder = new StringBuilder();
String line;
boolean verbatim = false;
private static void expandDataFileRules(Path file) throws IOException {
boolean modified = false;
int lineNum = 0;
try {
StringBuilder builder = new StringBuilder();
try (InputStream stream = Files.newInputStream(file);
InputStreamReader reader = new InputStreamReader(stream, StandardCharsets.UTF_8);
BufferedReader bufferedReader = new BufferedReader(reader)) {
String line;
boolean verbatim = false;
int lineNum = 0;
while (null != (line = bufferedReader.readLine())) {
++lineNum;
if (VERBATIM_RULE_LINE_PATTERN.matcher(line).matches()) {
@ -124,7 +125,7 @@ public class GenerateUTR30DataFiles {
String rightHandSide = ruleMatcher.group(2).trim();
expandSingleRule(builder, leftHandSide, rightHandSide);
} catch (IllegalArgumentException e) {
System.err.println("ERROR in " + file.getName() + " line #" + lineNum + ":");
System.err.println("ERROR in " + file.getFileName() + " line #" + lineNum + ":");
e.printStackTrace(System.err);
System.exit(1);
}
@ -142,18 +143,11 @@ public class GenerateUTR30DataFiles {
}
}
}
} finally {
bufferedReader.close();
}
if (modified) {
System.err.println("Expanding rules in and overwriting " + file.getName());
final FileOutputStream out = new FileOutputStream(file, false);
Writer writer = new OutputStreamWriter(out, StandardCharsets.UTF_8);
try {
writer.write(builder.toString());
} finally {
writer.close();
}
System.err.println("Expanding rules in and overwriting " + file.getFileName());
Files.writeString(file, builder.toString(), StandardCharsets.UTF_8);
}
}
@ -171,11 +165,12 @@ public class GenerateUTR30DataFiles {
System.err.print("Downloading " + NFKC_CF_TXT + " and making diacritic rules one-way ... ");
URLConnection connection = openConnection(new URL(norm2url, NFC_TXT));
BufferedReader reader =
new BufferedReader(
new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8));
Writer writer = new OutputStreamWriter(new FileOutputStream(NFC_TXT), StandardCharsets.UTF_8);
try {
try (BufferedReader reader =
new BufferedReader(
new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8));
Writer writer =
new OutputStreamWriter(
Files.newOutputStream(Path.of(NFC_TXT)), StandardCharsets.UTF_8)) {
String line;
while (null != (line = reader.readLine())) {
@ -208,9 +203,6 @@ public class GenerateUTR30DataFiles {
writer.write(line);
writer.write("\n");
}
} finally {
reader.close();
writer.close();
}
System.err.println("done.");
}
@ -218,7 +210,7 @@ public class GenerateUTR30DataFiles {
private static void download(URL url, String outputFile) throws IOException {
final URLConnection connection = openConnection(url);
final InputStream inputStream = connection.getInputStream();
final OutputStream outputStream = new FileOutputStream(outputFile);
final OutputStream outputStream = Files.newOutputStream(Path.of(outputFile));
int numBytes;
try {
while (-1 != (numBytes = inputStream.read(DOWNLOAD_BUFFER))) {

51
lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/RBBIRuleCompiler.java
View File

@ -18,14 +18,16 @@ package org.apache.lucene.analysis.icu;
import com.ibm.icu.text.RuleBasedBreakIterator;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.stream.Collectors;
/**
* Command-line utility to converts RuleBasedBreakIterator (.rbbi) files into binary compiled form
@ -33,9 +35,9 @@ import java.nio.charset.StandardCharsets;
*/
public class RBBIRuleCompiler {
static String getRules(File ruleFile) throws IOException {
static String getRules(Path ruleFile) throws IOException {
StringBuilder rules = new StringBuilder();
InputStream in = new FileInputStream(ruleFile);
InputStream in = Files.newInputStream(ruleFile);
BufferedReader cin = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
String line = null;
while ((line = cin.readLine()) != null) {
@ -49,20 +51,21 @@ public class RBBIRuleCompiler {
return rules.toString();
}
static void compile(File srcDir, File destDir) throws Exception {
File files[] =
srcDir.listFiles(
new FilenameFilter() {
public boolean accept(File dir, String name) {
return name.endsWith("rbbi");
}
});
if (files == null) throw new IOException("Path does not exist: " + srcDir);
for (int i = 0; i < files.length; i++) {
File file = files[i];
File outputFile = new File(destDir, file.getName().replaceAll("rbbi$", "brk"));
static void compile(Path srcDir, Path destDir) throws Exception {
List<Path> files;
try (var stream = Files.list(srcDir)) {
files =
stream
.filter(name -> name.getFileName().toString().endsWith("rbbi"))
.collect(Collectors.toList());
}
if (files.isEmpty()) throw new IOException("No input files matching *.rbbi at: " + srcDir);
for (Path file : files) {
Path outputFile = destDir.resolve(file.getFileName().toString().replaceAll("rbbi$", "brk"));
String rules = getRules(file);
System.err.print("Compiling " + file.getName() + " to " + outputFile.getName() + ": ");
System.err.print(
"Compiling " + file.getFileName() + " to " + outputFile.getFileName() + ": ");
/*
* if there is a syntax error, compileRules() may succeed. the way to
* check is to try to instantiate from the string. additionally if the
@ -78,10 +81,10 @@ public class RBBIRuleCompiler {
System.err.println(e.getMessage());
System.exit(1);
}
FileOutputStream os = new FileOutputStream(outputFile);
RuleBasedBreakIterator.compileRules(rules, os);
os.close();
System.err.println(outputFile.length() + " bytes.");
try (OutputStream os = Files.newOutputStream(outputFile)) {
RuleBasedBreakIterator.compileRules(rules, os);
}
System.err.println(Files.size(outputFile) + " bytes.");
}
}
@ -90,7 +93,7 @@ public class RBBIRuleCompiler {
System.err.println("Usage: RBBIRuleComputer <sourcedir> <destdir>");
System.exit(1);
}
compile(new File(args[0]), new File(args[1]));
compile(Paths.get(args[0]), Paths.get(args[1]));
System.exit(0);
}
}