Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-9760: Hunspell: print total memory usage in TestAllDictionaries, cleanup (#2347)

This commit is contained in:
Peter Gromov 2021-02-11 09:11:09 +01:00 committed by GitHub
parent 73d4f8cd03
commit c0d32444be
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 32 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
View File

@ -37,6 +37,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -44,6 +45,7 @@ import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.RamUsageTester;
import org.junit.Assert;
import org.junit.Assume;
@ -147,28 +149,33 @@ public class TestAllDictionaries extends LuceneTestCase {
}
public void testDictionariesLoadSuccessfully() throws Exception {
AtomicLong totalMemory = new AtomicLong();
AtomicLong totalWords = new AtomicLong();
int threads = Runtime.getRuntime().availableProcessors();
ExecutorService executor =
Executors.newFixedThreadPool(threads, new NamedThreadFactory("dictCheck-"));
try {
List<Path> failures = Collections.synchronizedList(new ArrayList<>());
Function<Path, Void> process =
(Path aff) -> {
try {
System.out.println(aff + "\t" + memoryUsage(loadDictionary(aff)));
} catch (Throwable e) {
failures.add(aff);
System.err.println("While checking " + aff + ":");
e.printStackTrace();
}
return null;
};
List<Path> failures = Collections.synchronizedList(new ArrayList<>());
Function<Path, Void> process =
(Path aff) -> {
try {
Dictionary dic = loadDictionary(aff);
totalMemory.addAndGet(RamUsageTester.sizeOf(dic));
totalWords.addAndGet(RamUsageTester.sizeOf(dic.words));
System.out.println(aff + "\t" + memoryUsageSummary(dic));
} catch (Throwable e) {
failures.add(aff);
System.err.println("While checking " + aff + ":");
e.printStackTrace();
}
return null;
};
for (Future<?> future :
executor.invokeAll(
findAllAffixFiles()
.map(aff -> (Callable<?>) () -> process.apply(aff))
.collect(Collectors.toList()))) {
List<Callable<Void>> tasks =
findAllAffixFiles()
.map(aff -> (Callable<Void>) () -> process.apply(aff))
.collect(Collectors.toList());
try {
for (Future<?> future : executor.invokeAll(tasks)) {
future.get();
}
@ -181,11 +188,16 @@ public class TestAllDictionaries extends LuceneTestCase {
}
} finally {
executor.shutdown();
executor.awaitTermination(1, TimeUnit.MINUTES);
assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
}
System.out.println("Total dictionaries loaded: " + tasks.size());
System.out.println("Total memory: " + RamUsageEstimator.humanReadableUnits(totalMemory.get()));
System.out.println(
"Total memory for word storage: " + RamUsageEstimator.humanReadableUnits(totalWords.get()));
}
private static String memoryUsage(Dictionary dic) {
private static String memoryUsageSummary(Dictionary dic) {
return RamUsageTester.humanSizeOf(dic)
+ "\t("
+ ("words=" + RamUsageTester.humanSizeOf(dic.words) + ", ")